From 1f6eeb89a9864f1af52255845907294c35fa8a0e Mon Sep 17 00:00:00 2001 From: quexeky <116044207+quexeky@users.noreply.github.com> Date: Tue, 1 Apr 2025 22:17:34 +1100 Subject: [PATCH 01/50] Initial commit --- libraries/droplet/.gitignore | 17 + libraries/droplet/LICENSE | 661 +++++++++++++++++++++++++++++++++++ libraries/droplet/README.md | 2 + 3 files changed, 680 insertions(+) create mode 100644 libraries/droplet/.gitignore create mode 100644 libraries/droplet/LICENSE create mode 100644 libraries/droplet/README.md diff --git a/libraries/droplet/.gitignore b/libraries/droplet/.gitignore new file mode 100644 index 00000000..0104787a --- /dev/null +++ b/libraries/droplet/.gitignore @@ -0,0 +1,17 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# RustRover +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/libraries/droplet/LICENSE b/libraries/droplet/LICENSE new file mode 100644 index 00000000..0ad25db4 --- /dev/null +++ b/libraries/droplet/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/libraries/droplet/README.md b/libraries/droplet/README.md new file mode 100644 index 00000000..b321a75e --- /dev/null +++ b/libraries/droplet/README.md @@ -0,0 +1,2 @@ +# droplet-rs +Core rust functionality for droplet From 961cbdcc9ec4293338e9a33a93f665d669761453 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:19:32 +1100 Subject: [PATCH 02/50] feat: Migrate all features to droplet-rs Signed-off-by: quexeky --- libraries/droplet/.gitignore | 6 +- libraries/droplet/Cargo.lock | 988 ++++++++++++++++++++++++++++ libraries/droplet/Cargo.toml | 50 ++ libraries/droplet/src/file_utils.rs | 24 + libraries/droplet/src/lib.rs | 4 + libraries/droplet/src/ssl.rs | 130 ++++ 6 files changed, 1201 insertions(+), 1 deletion(-) create mode 100644 libraries/droplet/Cargo.lock create mode 100644 libraries/droplet/Cargo.toml create mode 100644 libraries/droplet/src/file_utils.rs create mode 100644 libraries/droplet/src/lib.rs create mode 100644 libraries/droplet/src/ssl.rs diff --git a/libraries/droplet/.gitignore b/libraries/droplet/.gitignore index 0104787a..423b4790 100644 --- a/libraries/droplet/.gitignore +++ b/libraries/droplet/.gitignore @@ -14,4 +14,8 @@ target/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +# Added by cargo + +/target diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock new file mode 100644 index 00000000..30051b61 --- /dev/null +++ b/libraries/droplet/Cargo.lock @@ -0,0 +1,988 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.12", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "cc" +version = "1.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "data-encoding" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.1", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "droplet-rs" +version = "0.7.0" +dependencies = [ + "hex", + "md5", + "napi", + "napi-build", + "napi-derive", + "rcgen", + "ring", + "serde", + "serde_json", + "time", + "time-macros", + "uuid", + "webpki", + "x509-parser 0.17.0", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "napi" +version = "2.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3" +dependencies = [ + "bitflags", + "ctor", + "napi-derive", + "napi-sys", + "once_cell", + "tokio", +] + +[[package]] +name = "napi-build" +version = "2.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28acfa557c083f6e254a786e01ba253fc56f18ee000afcd4f79af735f73a6da" + +[[package]] +name = "napi-derive" +version = "2.16.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" +dependencies = [ + "cfg-if", + "convert_case", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" +dependencies = [ + "convert_case", + "once_cell", + "proc-macro2", + "quote", + "regex", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" +dependencies = [ + "libloading", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.1", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64", + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha", + "rand_core", + "zerocopy", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser 0.16.0", + "yasna", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tokio" +version = "1.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +dependencies = [ + "backtrace", + "pin-project-lite", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.2", + "rand", + "uuid-macro-internal", +] + +[[package]] +name = "uuid-macro-internal" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72dcd78c4f979627a754f5522cea6e6a25e55139056535fe6e69c506cd64a862" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "ring", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" +dependencies = [ + "asn1-rs 0.7.1", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "ring", + "rusticata-macros", + "thiserror 2.0.12", + "time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml new file mode 100644 index 00000000..5f44c057 --- /dev/null +++ b/libraries/droplet/Cargo.toml @@ -0,0 +1,50 @@ +[package] +edition = "2024" +name = "droplet-rs" +version = "0.7.0" +license = "AGPL-3.0-only" +description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix +napi = { version = "2.12.2", default-features = false, features = [ + "napi4", + "async", +] } +napi-derive = "2.12.2" +hex = "0.4.3" +serde_json = "1.0.128" +md5 = "0.7.0" +time-macros = "0.2.22" +time = "0.3.41" +webpki = "0.22.4" +ring = "0.17.14" + +[dependencies.x509-parser] +version = "0.17.0" +features = ["verify"] + +[dependencies.rcgen] +version = "0.13.2" +features = ["crypto", "pem", "x509-parser"] + +[dependencies.serde] +version = "1.0.210" +features = ["serde_derive"] + +[dependencies.uuid] +version = "1.10.0" +features = [ + "v4", # Lets you generate random UUIDs + "fast-rng", # Use a faster (but still sufficiently random) RNG + "macro-diagnostics", # Enable better diagnostics for compile-time UUIDs +] + +[build-dependencies] +napi-build = "2.0.1" + +[profile.release] +lto = true +strip = "symbols" \ No newline at end of file diff --git a/libraries/droplet/src/file_utils.rs b/libraries/droplet/src/file_utils.rs new file mode 100644 index 00000000..332e61ff --- /dev/null +++ b/libraries/droplet/src/file_utils.rs @@ -0,0 +1,24 @@ +use std::{ + fs::{self, metadata}, + path::{Path, PathBuf}, +}; + +fn list_files_recursive(vec: &mut Vec, path: &Path) { + if metadata(path).unwrap().is_dir() { + let paths = fs::read_dir(path).unwrap(); + for path_result in paths { + let full_path = path_result.unwrap().path(); + if metadata(&full_path).unwrap().is_dir() { + list_files_recursive(vec, &full_path); + } else { + vec.push(full_path); + } + } + } +} + +pub fn list_files(path: &Path) -> Vec { + let mut vec = Vec::new(); + list_files_recursive(&mut vec, path); + vec +} diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs new file mode 100644 index 00000000..fc5931d1 --- /dev/null +++ b/libraries/droplet/src/lib.rs @@ -0,0 +1,4 @@ +#![deny(clippy::all)] + +pub mod file_utils; +pub mod ssl; diff --git a/libraries/droplet/src/ssl.rs b/libraries/droplet/src/ssl.rs new file mode 100644 index 00000000..fb6e56ab --- /dev/null +++ b/libraries/droplet/src/ssl.rs @@ -0,0 +1,130 @@ +use napi::Error; +use rcgen::{ + CertificateParams, DistinguishedName, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData, + SubjectPublicKeyInfo, +}; +use ring::rand::SystemRandom; +use ring::signature::{EcdsaKeyPair, VerificationAlgorithm}; +use time::{Duration, OffsetDateTime}; +use x509_parser::parse_x509_certificate; +use x509_parser::pem::Pem; + +pub fn generate_root_ca() -> Result, Error> { + let mut params = CertificateParams::default(); + + let mut name = DistinguishedName::new(); + name.push(rcgen::DnType::CommonName, "Drop Root Server"); + name.push(rcgen::DnType::OrganizationName, "Drop"); + + params.distinguished_name = name; + + params.not_before = OffsetDateTime::now_utc(); + params.not_after = OffsetDateTime::now_utc() + .checked_add(Duration::days(365 * 1000)) + .unwrap(); + + params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + + params.key_usages = vec![ + KeyUsagePurpose::CrlSign, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::DigitalSignature, + ]; + + let key_pair = KeyPair::generate().map_err(|e| napi::Error::from_reason(e.to_string()))?; + let certificate = CertificateParams::self_signed(params, &key_pair) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + + // Returns certificate, then private key + Ok(vec![certificate.pem(), key_pair.serialize_pem()]) +} + +pub fn generate_client_certificate( + client_id: String, + _client_name: String, + root_ca: String, + root_ca_private: String, +) -> Result, Error> { + let root_key_pair = + KeyPair::from_pem(&root_ca_private).map_err(|e| napi::Error::from_reason(e.to_string()))?; + let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + + let mut params = CertificateParams::default(); + + let mut name = DistinguishedName::new(); + name.push(rcgen::DnType::CommonName, client_id); + name.push(rcgen::DnType::OrganizationName, "Drop"); + params.distinguished_name = name; + + params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::DataEncipherment, + ]; + + let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + + // Returns certificate, then private key + Ok(vec![certificate.pem(), key_pair.serialize_pem()]) +} + +pub fn verify_client_certificate(client_cert: String, root_ca: String) -> Result { + let root_ca = Pem::iter_from_buffer(root_ca.as_bytes()) + .next() + .unwrap() + .unwrap(); + let root_ca = root_ca.parse_x509().unwrap(); + + let client_cert = Pem::iter_from_buffer(client_cert.as_bytes()) + .next() + .unwrap() + .unwrap(); + let client_cert = client_cert.parse_x509().unwrap(); + + let valid = root_ca + .verify_signature(Some(client_cert.public_key())) + .is_ok(); + + Ok(valid) +} + +pub fn sign_nonce(private_key: String, nonce: String) -> Result { + let rng = SystemRandom::new(); + + let key_pair = KeyPair::from_pem(&private_key).unwrap(); + + let key_pair = EcdsaKeyPair::from_pkcs8( + &ring::signature::ECDSA_P384_SHA384_FIXED_SIGNING, + &key_pair.serialize_der(), + &rng, + ) + .unwrap(); + + let signature = key_pair.sign(&rng, nonce.as_bytes()).unwrap(); + let hex_signature = hex::encode(signature); + + Ok(hex_signature) +} + +pub fn verify_nonce(public_cert: String, nonce: String, signature: String) -> Result { + let (_, pem) = x509_parser::pem::parse_x509_pem(public_cert.as_bytes()).unwrap(); + let (_, spki) = parse_x509_certificate(&pem.contents).unwrap(); + let public_key = SubjectPublicKeyInfo::from_der(spki.public_key().raw).unwrap(); + + let raw_signature = hex::decode(signature).unwrap(); + + let valid = ring::signature::ECDSA_P384_SHA384_FIXED + .verify( + public_key.der_bytes().into(), + nonce.as_bytes().into(), + raw_signature[..].into(), + ) + .is_ok(); + + Ok(valid) +} From 0b8edf7bc292889e3c64e5613e989ab2e72afb10 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:26:31 +1100 Subject: [PATCH 03/50] chore: Update authors Signed-off-by: quexeky --- libraries/droplet/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 5f44c057..76ad9a13 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -1,5 +1,6 @@ [package] edition = "2024" +authors = ["Drop-OSS"] name = "droplet-rs" version = "0.7.0" license = "AGPL-3.0-only" From 39e60587d26fffcb0fc64ba86c717fadd37eb028 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:46:02 +1100 Subject: [PATCH 04/50] refactor: Remove unused dependencies Signed-off-by: quexeky --- libraries/droplet/Cargo.lock | 196 +---------------------------------- libraries/droplet/Cargo.toml | 15 --- 2 files changed, 2 insertions(+), 209 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 30051b61..6bed7620 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -17,15 +17,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - [[package]] name = "asn1-rs" version = "0.6.2" @@ -219,17 +210,12 @@ name = "droplet-rs" version = "0.7.0" dependencies = [ "hex", - "md5", "napi", "napi-build", - "napi-derive", "rcgen", "ring", - "serde", - "serde_json", "time", "time-macros", - "uuid", "webpki", "x509-parser 0.17.0", ] @@ -242,19 +228,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasi", ] [[package]] @@ -297,12 +271,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - [[package]] name = "memchr" version = "2.7.4" @@ -368,8 +336,6 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "regex", - "semver", "syn", ] @@ -481,15 +447,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "proc-macro2" version = "1.0.94" @@ -508,42 +465,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - -[[package]] -name = "rand" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" -dependencies = [ - "rand_chacha", - "rand_core", - "zerocopy", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.2", -] - [[package]] name = "rcgen" version = "0.13.2" @@ -558,35 +479,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - [[package]] name = "ring" version = "0.17.14" @@ -595,7 +487,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom", "libc", "untrusted", "windows-sys", @@ -622,18 +514,6 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - [[package]] name = "serde" version = "1.0.219" @@ -654,18 +534,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_json" -version = "1.0.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - [[package]] name = "shlex" version = "1.3.0" @@ -793,43 +661,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -dependencies = [ - "getrandom 0.3.2", - "rand", - "uuid-macro-internal", -] - -[[package]] -name = "uuid-macro-internal" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72dcd78c4f979627a754f5522cea6e6a25e55139056535fe6e69c506cd64a862" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - [[package]] name = "webpki" version = "0.22.4" @@ -913,15 +750,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - [[package]] name = "x509-parser" version = "0.16.0" @@ -966,23 +794,3 @@ checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ "time", ] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 76ad9a13..a0298015 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -14,10 +14,7 @@ napi = { version = "2.12.2", default-features = false, features = [ "napi4", "async", ] } -napi-derive = "2.12.2" hex = "0.4.3" -serde_json = "1.0.128" -md5 = "0.7.0" time-macros = "0.2.22" time = "0.3.41" webpki = "0.22.4" @@ -31,18 +28,6 @@ features = ["verify"] version = "0.13.2" features = ["crypto", "pem", "x509-parser"] -[dependencies.serde] -version = "1.0.210" -features = ["serde_derive"] - -[dependencies.uuid] -version = "1.10.0" -features = [ - "v4", # Lets you generate random UUIDs - "fast-rng", # Use a faster (but still sufficiently random) RNG - "macro-diagnostics", # Enable better diagnostics for compile-time UUIDs -] - [build-dependencies] napi-build = "2.0.1" From bab8a294945cbcba89f2e6e980bd8854377dad85 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:47:42 +1100 Subject: [PATCH 05/50] fix: Rollback to 2021 edition Signed-off-by: quexeky --- libraries/droplet/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index a0298015..6250977e 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -1,5 +1,5 @@ [package] -edition = "2024" +edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" version = "0.7.0" From 14405f70099fc3492002ad8155642b8a25bcd60b Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:48:12 +1100 Subject: [PATCH 06/50] chore: Version bump Signed-off-by: quexeky --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 6bed7620..7feb1c7a 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -207,7 +207,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.7.0" +version = "0.7.1" dependencies = [ "hex", "napi", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 6250977e..06f0ed20 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.7.0" +version = "0.7.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" [lib] From 8c209672e06604f197ffbf01ffe9494c3d569216 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:53:20 +1100 Subject: [PATCH 07/50] fix: Remove napi dependencies Signed-off-by: quexeky --- libraries/droplet/Cargo.lock | 181 ----------------------------------- libraries/droplet/Cargo.toml | 10 +- libraries/droplet/src/ssl.rs | 27 ++---- 3 files changed, 11 insertions(+), 207 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 7feb1c7a..4c7126b7 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - [[package]] name = "asn1-rs" version = "0.6.2" @@ -90,33 +75,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bitflags" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" - [[package]] name = "cc" version = "1.2.17" @@ -132,25 +96,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "convert_case" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "ctor" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "data-encoding" version = "2.8.0" @@ -210,8 +155,6 @@ name = "droplet-rs" version = "0.7.1" dependencies = [ "hex", - "napi", - "napi-build", "rcgen", "ring", "time", @@ -231,12 +174,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "hex" version = "0.4.3" @@ -261,16 +198,6 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" -[[package]] -name = "libloading" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" -dependencies = [ - "cfg-if", - "windows-targets", -] - [[package]] name = "memchr" version = "2.7.4" @@ -283,71 +210,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -dependencies = [ - "adler2", -] - -[[package]] -name = "napi" -version = "2.16.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3" -dependencies = [ - "bitflags", - "ctor", - "napi-derive", - "napi-sys", - "once_cell", - "tokio", -] - -[[package]] -name = "napi-build" -version = "2.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28acfa557c083f6e254a786e01ba253fc56f18ee000afcd4f79af735f73a6da" - -[[package]] -name = "napi-derive" -version = "2.16.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" -dependencies = [ - "cfg-if", - "convert_case", - "napi-derive-backend", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "napi-derive-backend" -version = "1.0.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" -dependencies = [ - "convert_case", - "once_cell", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "napi-sys" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" -dependencies = [ - "libloading", -] - [[package]] name = "nom" version = "7.1.3" @@ -392,15 +254,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "oid-registry" version = "0.7.1" @@ -419,12 +272,6 @@ dependencies = [ "asn1-rs 0.7.1", ] -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - [[package]] name = "pem" version = "3.0.5" @@ -435,12 +282,6 @@ dependencies = [ "serde", ] -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - [[package]] name = "powerfmt" version = "0.2.0" @@ -493,12 +334,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - [[package]] name = "rusticata-macros" version = "4.1.0" @@ -633,28 +468,12 @@ dependencies = [ "time-core", ] -[[package]] -name = "tokio" -version = "1.44.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" -dependencies = [ - "backtrace", - "pin-project-lite", -] - [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 06f0ed20..c1a4b0ae 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -9,11 +9,6 @@ description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performa crate-type = ["cdylib"] [dependencies] -# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix -napi = { version = "2.12.2", default-features = false, features = [ - "napi4", - "async", -] } hex = "0.4.3" time-macros = "0.2.22" time = "0.3.41" @@ -28,9 +23,6 @@ features = ["verify"] version = "0.13.2" features = ["crypto", "pem", "x509-parser"] -[build-dependencies] -napi-build = "2.0.1" - [profile.release] lto = true -strip = "symbols" \ No newline at end of file +strip = "symbols" diff --git a/libraries/droplet/src/ssl.rs b/libraries/droplet/src/ssl.rs index fb6e56ab..6f59e984 100644 --- a/libraries/droplet/src/ssl.rs +++ b/libraries/droplet/src/ssl.rs @@ -1,7 +1,5 @@ -use napi::Error; use rcgen::{ - CertificateParams, DistinguishedName, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData, - SubjectPublicKeyInfo, + CertificateParams, DistinguishedName, Error, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData, SubjectPublicKeyInfo }; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, VerificationAlgorithm}; @@ -9,7 +7,7 @@ use time::{Duration, OffsetDateTime}; use x509_parser::parse_x509_certificate; use x509_parser::pem::Pem; -pub fn generate_root_ca() -> Result, Error> { +pub fn generate_root_ca() -> Result, rcgen::Error> { let mut params = CertificateParams::default(); let mut name = DistinguishedName::new(); @@ -31,9 +29,8 @@ pub fn generate_root_ca() -> Result, Error> { KeyUsagePurpose::DigitalSignature, ]; - let key_pair = KeyPair::generate().map_err(|e| napi::Error::from_reason(e.to_string()))?; - let certificate = CertificateParams::self_signed(params, &key_pair) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + let key_pair = KeyPair::generate()?; + let certificate = CertificateParams::self_signed(params, &key_pair)?; // Returns certificate, then private key Ok(vec![certificate.pem(), key_pair.serialize_pem()]) @@ -44,13 +41,11 @@ pub fn generate_client_certificate( _client_name: String, root_ca: String, root_ca_private: String, -) -> Result, Error> { +) -> Result, rcgen::Error> { let root_key_pair = - KeyPair::from_pem(&root_ca_private).map_err(|e| napi::Error::from_reason(e.to_string()))?; - let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; - let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + KeyPair::from_pem(&root_ca_private)?; + let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)?; + let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)?; let mut params = CertificateParams::default(); @@ -64,10 +59,8 @@ pub fn generate_client_certificate( KeyUsagePurpose::DataEncipherment, ]; - let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; - let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + let key_pair = KeyPair::generate_for(&rcgen::PKCS_ECDSA_P384_SHA384)?; + let certificate = CertificateParams::signed_by(params, &key_pair, &root_ca, &root_key_pair)?; // Returns certificate, then private key Ok(vec![certificate.pem(), key_pair.serialize_pem()]) From 4e9794f628d46068ec096293fb6e61b386b84509 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 09:53:33 +1100 Subject: [PATCH 08/50] chore: Version bump Signed-off-by: quexeky --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 4c7126b7..4ff84d04 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -152,7 +152,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.7.1" +version = "0.7.2" dependencies = [ "hex", "rcgen", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index c1a4b0ae..9c3cb684 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.7.1" +version = "0.7.2" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" [lib] From b2de86b8920a5a1080993f4616069ec64ea27546 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 10:04:18 +1100 Subject: [PATCH 09/50] fix: Removed cdylib reference Signed-off-by: quexeky --- libraries/droplet/Cargo.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 9c3cb684..222296a0 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -1,12 +1,10 @@ -[package] +[lib] edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.7.2" +version = "0.7.3" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" -[lib] -crate-type = ["cdylib"] [dependencies] hex = "0.4.3" From b0df1a8b42d8dd707ad2a1766f56b1a0ef6e2dbf Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 2 Apr 2025 10:10:47 +1100 Subject: [PATCH 10/50] fix: Use [package] instead of [lib] Signed-off-by: quexeky --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 4ff84d04..6d36edbb 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -152,7 +152,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.7.2" +version = "0.7.3" dependencies = [ "hex", "rcgen", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 222296a0..bb08d453 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -1,4 +1,4 @@ -[lib] +[package] edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" From 2d67febdb1aa96f79036301554a8466d464a338f Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 11:11:46 +1100 Subject: [PATCH 11/50] feat: move version backend --- libraries/droplet/Cargo.lock | 32 ++- libraries/droplet/Cargo.toml | 5 +- libraries/droplet/src/lib.rs | 1 + libraries/droplet/src/versions/backends.rs | 246 +++++++++++++++++++++ libraries/droplet/src/versions/mod.rs | 2 + libraries/droplet/src/versions/types.rs | 55 +++++ 6 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 libraries/droplet/src/versions/backends.rs create mode 100644 libraries/droplet/src/versions/mod.rs create mode 100644 libraries/droplet/src/versions/types.rs diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 6d36edbb..22be2c74 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "asn1-rs" version = "0.6.2" @@ -152,17 +158,26 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.7.3" +version = "0.8.0" dependencies = [ + "anyhow", + "dyn-clone", "hex", "rcgen", "ring", "time", "time-macros", + "tokio", "webpki", "x509-parser 0.17.0", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "getrandom" version = "0.2.15" @@ -282,6 +297,12 @@ dependencies = [ "serde", ] +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "powerfmt" version = "0.2.0" @@ -468,6 +489,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "pin-project-lite", +] + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index bb08d453..b0d0c7da 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.7.3" +version = "0.8.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -12,6 +12,9 @@ time-macros = "0.2.22" time = "0.3.41" webpki = "0.22.4" ring = "0.17.14" +dyn-clone = "1.0.20" +tokio = "1.48.0" +anyhow = "1.0.100" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index fc5931d1..ec1ad139 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -2,3 +2,4 @@ pub mod file_utils; pub mod ssl; +pub mod versions; diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs new file mode 100644 index 00000000..96fd6ad8 --- /dev/null +++ b/libraries/droplet/src/versions/backends.rs @@ -0,0 +1,246 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{ + cell::LazyCell, + fs::{self, metadata, File}, + io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink}, + path::{Path, PathBuf}, + process::{Child, ChildStdout, Command, Stdio}, + sync::{Arc, LazyLock}, +}; + +use anyhow::{anyhow, Result}; + +use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; + +pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { + if metadata(path)?.is_dir() { + let paths = fs::read_dir(path)?; + for path_result in paths { + let full_path = path_result?.path(); + if metadata(&full_path)?.is_dir() { + _list_files(vec, &full_path)?; + } else { + vec.push(full_path); + } + } + }; + + Ok(()) +} + +#[derive(Clone)] +pub struct PathVersionBackend { + pub base_dir: PathBuf, +} +impl VersionBackend for PathVersionBackend { + fn list_files(&mut self) -> anyhow::Result> { + let mut vec = Vec::new(); + _list_files(&mut vec, &self.base_dir)?; + + let mut results = Vec::new(); + + for pathbuf in vec.iter() { + let relative = pathbuf.strip_prefix(self.base_dir.clone())?; + + results.push( + self.peek_file( + relative + .to_str() + .ok_or(anyhow!("Could not parse path"))? + .to_owned(), + )?, + ); + } + + Ok(results) + } + + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?; + + if start != 0 { + file.seek(SeekFrom::Start(start))?; + } + + if end != 0 { + return Ok(Box::new(file.take(end - start))); + } + + Ok(Box::new(file)) + } + + fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let pathbuf = self.base_dir.join(sub_path.clone()); + if !pathbuf.exists() { + return Err(anyhow!("Path doesn't exist.")); + }; + + let file = File::open(pathbuf.clone())?; + let metadata = file.try_clone()?.metadata()?; + let permission_object = metadata.permissions(); + let permissions = { + let perm: u32; + #[cfg(target_family = "unix")] + { + perm = permission_object.mode(); + } + #[cfg(not(target_family = "unix"))] + { + perm = 0 + } + perm + }; + + Ok(VersionFile { + relative_filename: sub_path, + permission: permissions, + size: metadata.len(), + }) + } + + fn require_whole_files(&self) -> bool { + false + } +} + +pub static SEVEN_ZIP_INSTALLED: LazyLock = + LazyLock::new(|| Command::new("7z").output().is_ok()); +// https://7-zip.opensource.jp/chm/general/formats.htm +// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above +pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ + "7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz", + "txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a", + "deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt", + "cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx", + "hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz", + "mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c", + "squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z", + "taz", +]; + +#[derive(Clone)] +pub struct ZipVersionBackend { + path: String, +} +impl ZipVersionBackend { + pub fn new(path: PathBuf) -> anyhow::Result { + Ok(Self { + path: path.to_str().expect("invalid utf path").to_owned(), + }) + } +} + +pub struct ZipFileWrapper { + command: Child, + reader: BufReader, +} + +impl ZipFileWrapper { + pub fn new(mut command: Child) -> Self { + let stdout = command + .stdout + .take() + .expect("failed to access stdout of 7z"); + let reader = BufReader::new(stdout); + ZipFileWrapper { command, reader } + } +} + +/** + * This read implemention is a result of debugging hell + * It should probably be replaced with a .take() call. + */ +impl Read for ZipFileWrapper { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reader.read(buf) + } +} + +impl Drop for ZipFileWrapper { + fn drop(&mut self) { + self.command.wait().expect("failed to wait for 7z exit"); + } +} + +impl VersionBackend for ZipVersionBackend { + fn list_files(&mut self) -> anyhow::Result> { + let mut list_command = Command::new("7z"); + list_command.args(vec!["l", "-ba", &self.path]); + let result = list_command.output()?; + if !result.status.success() { + return Err(anyhow!( + "failed to list files: code {:?}", + result.status.code() + )); + } + let raw_result = String::from_utf8(result.stdout)?; + let files = raw_result + .split("\n") + .filter(|v| v.len() > 0) + .map(|v| v.split(" ").filter(|v| v.len() > 0)); + let mut results = Vec::new(); + + for file in files { + let values = file.collect::>(); + let mut iter = values.iter(); + let (date, time, attrs, size, compress, name) = ( + iter.next().expect("failed to read date"), + iter.next().expect("failed to read time"), + iter.next().expect("failed to read attrs"), + iter.next().expect("failed to read size"), + iter.next().expect("failed to read compress"), + iter.collect::>(), + ); + if attrs.starts_with("D") { + continue; + } + results.push(VersionFile { + relative_filename: name + .into_iter() + .map(|v| *v) + .fold(String::new(), |a, b| a + b + " ") + .trim_end() + .to_owned(), + permission: 0o744, // owner r/w/x, everyone else, read + size: size.parse().unwrap(), + }); + } + + Ok(results) + } + + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut read_command = Command::new("7z"); + read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); + let output = read_command + .stdout(Stdio::piped()) + .spawn() + .expect("failed to spawn 7z"); + Ok(Box::new(ZipFileWrapper::new(output))) + } + + fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let files = self.list_files()?; + let file = files + .iter() + .find(|v| v.relative_filename == sub_path) + .expect("file not found"); + + Ok(file.clone()) + } + + fn require_whole_files(&self) -> bool { + true + } +} diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs new file mode 100644 index 00000000..21b07e7c --- /dev/null +++ b/libraries/droplet/src/versions/mod.rs @@ -0,0 +1,2 @@ +pub mod types; +pub mod backends; \ No newline at end of file diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs new file mode 100644 index 00000000..91c234d5 --- /dev/null +++ b/libraries/droplet/src/versions/types.rs @@ -0,0 +1,55 @@ +use std::{fmt::Debug, io::Read}; + +use dyn_clone::DynClone; +use tokio::io::{self, AsyncRead}; + +#[derive(Debug, Clone)] +pub struct VersionFile { + pub relative_filename: String, + pub permission: u32, + pub size: u64, +} + +pub trait MinimumFileObject: Read + Send {} +impl MinimumFileObject for T {} + +// Intentionally not a generic, because of types in read_file +pub struct ReadToAsyncRead<'a> { + pub inner: Box, +} + +const ASYNC_READ_BUFFER_SIZE: usize = 8128; + +impl<'a> AsyncRead for ReadToAsyncRead<'a> { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; + let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); + match self.inner.read(&mut read_buf[0..read_size]) { + Ok(read) => { + buf.put_slice(&read_buf[0..read]); + std::task::Poll::Ready(Ok(())) + } + Err(err) => { + std::task::Poll::Ready(Err(err)) + }, + } + } +} + +pub trait VersionBackend: DynClone { + fn require_whole_files(&self) -> bool; + fn list_files(&mut self) -> anyhow::Result>; + fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result>; +} + +dyn_clone::clone_trait_object!(VersionBackend); From 7c49a5cacb63dbf246a0e853db489c166fa0ea38 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 11:14:18 +1100 Subject: [PATCH 12/50] feat: add backend constructor --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/versions/mod.rs | 54 ++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 22be2c74..a88d8f7e 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -158,7 +158,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.8.0" +version = "0.8.1" dependencies = [ "anyhow", "dyn-clone", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index b0d0c7da..0ec263d7 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.8.0" +version = "0.8.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 21b07e7c..fd11106b 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -1,2 +1,54 @@ +use std::path::Path; + +use anyhow::Result; + +use crate::versions::{ + backends::{ + PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED, SUPPORTED_FILE_EXTENSIONS, + }, + types::VersionBackend, +}; + +pub mod backends; pub mod types; -pub mod backends; \ No newline at end of file +pub fn create_backend_constructor<'a>( + path: &Path, +) -> Option Result>>> { + if !path.exists() { + return None; + } + + let is_directory = path.is_dir(); + if is_directory { + let base_dir = path.to_path_buf(); + return Some(Box::new(move || { + Ok(Box::new(PathVersionBackend { base_dir })) + })); + }; + + if *SEVEN_ZIP_INSTALLED { + /* + Slow 7zip integrity test + let mut test = Command::new("7z"); + test.args(vec!["t", path.to_str().expect("invalid utf path")]); + let status = test.status().ok()?; + if status.code().unwrap_or(1) == 0 { + let buf = path.to_path_buf(); + return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); + } + */ + // Fast filename-based test + if let Some(extension) = path.extension().and_then(|v| v.to_str()) { + let supported = SUPPORTED_FILE_EXTENSIONS + .iter() + .find(|v| ***v == *extension) + .is_some(); + if supported { + let buf = path.to_path_buf(); + return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); + } + } + } + + None +} From 724415df78b41735b835ed917b0255bd71238083 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 11:25:53 +1100 Subject: [PATCH 13/50] feat: make version backends sync --- libraries/droplet/src/versions/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index fd11106b..19e001a9 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -13,7 +13,7 @@ pub mod backends; pub mod types; pub fn create_backend_constructor<'a>( path: &Path, -) -> Option Result>>> { +) -> Option Result>>> { if !path.exists() { return None; } From 217079cdf790d4b388b61e8090959b2dc1bca7f5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 14:29:41 +1100 Subject: [PATCH 14/50] feat: async update --- libraries/droplet/Cargo.lock | 62 ++++++++++++++- libraries/droplet/Cargo.toml | 5 +- libraries/droplet/src/versions/backends.rs | 92 +++++++++------------- libraries/droplet/src/versions/types.rs | 62 +++++++-------- 4 files changed, 131 insertions(+), 90 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index a88d8f7e..17f4ffc1 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -75,6 +75,17 @@ dependencies = [ "syn", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -87,6 +98,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + [[package]] name = "cc" version = "1.2.17" @@ -158,9 +175,10 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.8.1" +version = "0.9.0" dependencies = [ "anyhow", + "async-trait", "dyn-clone", "hex", "rcgen", @@ -225,6 +243,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nom" version = "7.1.3" @@ -352,7 +381,7 @@ dependencies = [ "getrandom", "libc", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -396,6 +425,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + [[package]] name = "syn" version = "2.0.100" @@ -495,7 +533,12 @@ version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "signal-hook-registry", + "windows-sys 0.61.2", ] [[package]] @@ -526,6 +569,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.52.0" @@ -535,6 +584,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 0ec263d7..12d2d69d 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.8.1" +version = "0.9.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -13,8 +13,9 @@ time = "0.3.41" webpki = "0.22.4" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = "1.48.0" +tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] } anyhow = "1.0.100" +async-trait = "0.1.89" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index 96fd6ad8..c42eb80a 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -1,21 +1,26 @@ #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ - cell::LazyCell, - fs::{self, metadata, File}, - io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink}, + fs::{metadata, read_dir}, + io::SeekFrom, path::{Path, PathBuf}, - process::{Child, ChildStdout, Command, Stdio}, - sync::{Arc, LazyLock}, + process::Stdio, + sync::LazyLock, }; use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use tokio::{ + fs::File, + io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, + process::Command, +}; use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { if metadata(path)?.is_dir() { - let paths = fs::read_dir(path)?; + let paths = read_dir(path)?; for path_result in paths { let full_path = path_result?.path(); if metadata(&full_path)?.is_dir() { @@ -33,8 +38,10 @@ pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { pub struct PathVersionBackend { pub base_dir: PathBuf, } + +#[async_trait] impl VersionBackend for PathVersionBackend { - fn list_files(&mut self) -> anyhow::Result> { + async fn list_files(&mut self) -> anyhow::Result> { let mut vec = Vec::new(); _list_files(&mut vec, &self.base_dir)?; @@ -49,23 +56,24 @@ impl VersionBackend for PathVersionBackend { .to_str() .ok_or(anyhow!("Could not parse path"))? .to_owned(), - )?, + ) + .await?, ); } Ok(results) } - fn reader( + async fn reader( &mut self, file: &VersionFile, start: u64, end: u64, - ) -> anyhow::Result> { - let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?; + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; if start != 0 { - file.seek(SeekFrom::Start(start))?; + file.seek(SeekFrom::Start(start)).await?; } if end != 0 { @@ -75,14 +83,14 @@ impl VersionBackend for PathVersionBackend { Ok(Box::new(file)) } - fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { let pathbuf = self.base_dir.join(sub_path.clone()); if !pathbuf.exists() { return Err(anyhow!("Path doesn't exist.")); }; - let file = File::open(pathbuf.clone())?; - let metadata = file.try_clone()?.metadata()?; + let file = File::open(pathbuf.clone()).await?; + let metadata = file.try_clone().await?.metadata().await?; let permission_object = metadata.permissions(); let permissions = { let perm: u32; @@ -110,7 +118,7 @@ impl VersionBackend for PathVersionBackend { } pub static SEVEN_ZIP_INSTALLED: LazyLock = - LazyLock::new(|| Command::new("7z").output().is_ok()); + LazyLock::new(|| std::process::Command::new("7z").output().is_ok()); // https://7-zip.opensource.jp/chm/general/formats.htm // Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ @@ -136,43 +144,12 @@ impl ZipVersionBackend { } } -pub struct ZipFileWrapper { - command: Child, - reader: BufReader, -} - -impl ZipFileWrapper { - pub fn new(mut command: Child) -> Self { - let stdout = command - .stdout - .take() - .expect("failed to access stdout of 7z"); - let reader = BufReader::new(stdout); - ZipFileWrapper { command, reader } - } -} - -/** - * This read implemention is a result of debugging hell - * It should probably be replaced with a .take() call. - */ -impl Read for ZipFileWrapper { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.reader.read(buf) - } -} - -impl Drop for ZipFileWrapper { - fn drop(&mut self) { - self.command.wait().expect("failed to wait for 7z exit"); - } -} - +#[async_trait] impl VersionBackend for ZipVersionBackend { - fn list_files(&mut self) -> anyhow::Result> { + async fn list_files(&mut self) -> anyhow::Result> { let mut list_command = Command::new("7z"); list_command.args(vec!["l", "-ba", &self.path]); - let result = list_command.output()?; + let result = list_command.output().await?; if !result.status.success() { return Err(anyhow!( "failed to list files: code {:?}", @@ -215,7 +192,7 @@ impl VersionBackend for ZipVersionBackend { Ok(results) } - fn reader( + async fn reader( &mut self, file: &VersionFile, start: u64, @@ -223,15 +200,20 @@ impl VersionBackend for ZipVersionBackend { ) -> anyhow::Result> { let mut read_command = Command::new("7z"); read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); - let output = read_command + let mut output = read_command .stdout(Stdio::piped()) .spawn() .expect("failed to spawn 7z"); - Ok(Box::new(ZipFileWrapper::new(output))) + let stdout = output + .stdout + .take() + .ok_or(anyhow!("no stdout on 7zip process"))?; + let reader = BufReader::new(stdout); + Ok(Box::new(reader)) } - fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let files = self.list_files()?; + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let files = self.list_files().await?; let file = files .iter() .find(|v| v.relative_filename == sub_path) diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 91c234d5..4fe2503c 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -1,55 +1,55 @@ use std::{fmt::Debug, io::Read}; +use async_trait::async_trait; use dyn_clone::DynClone; use tokio::io::{self, AsyncRead}; #[derive(Debug, Clone)] pub struct VersionFile { - pub relative_filename: String, - pub permission: u32, - pub size: u64, + pub relative_filename: String, + pub permission: u32, + pub size: u64, } -pub trait MinimumFileObject: Read + Send {} -impl MinimumFileObject for T {} +pub trait MinimumFileObject: AsyncRead + Send {} +impl MinimumFileObject for T {} // Intentionally not a generic, because of types in read_file pub struct ReadToAsyncRead<'a> { - pub inner: Box, + pub inner: Box, } const ASYNC_READ_BUFFER_SIZE: usize = 8128; impl<'a> AsyncRead for ReadToAsyncRead<'a> { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; - let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); - match self.inner.read(&mut read_buf[0..read_size]) { - Ok(read) => { - buf.put_slice(&read_buf[0..read]); - std::task::Poll::Ready(Ok(())) - } - Err(err) => { - std::task::Poll::Ready(Err(err)) - }, + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; + let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); + match self.inner.read(&mut read_buf[0..read_size]) { + Ok(read) => { + buf.put_slice(&read_buf[0..read]); + std::task::Poll::Ready(Ok(())) + } + Err(err) => std::task::Poll::Ready(Err(err)), + } } - } } +#[async_trait] pub trait VersionBackend: DynClone { - fn require_whole_files(&self) -> bool; - fn list_files(&mut self) -> anyhow::Result>; - fn peek_file(&mut self, sub_path: String) -> anyhow::Result; - fn reader( - &mut self, - file: &VersionFile, - start: u64, - end: u64, - ) -> anyhow::Result>; + fn require_whole_files(&self) -> bool; + async fn list_files(&mut self) -> anyhow::Result>; + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + async fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result>; } dyn_clone::clone_trait_object!(VersionBackend); From acdf2ba0f73dd0ce0dcda8491ca29dba0a3884cc Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 14:33:32 +1100 Subject: [PATCH 15/50] fix: add unpin type --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/versions/types.rs | 28 ++----------------------- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 17f4ffc1..2db3ae35 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -175,7 +175,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.9.0" +version = "0.9.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 12d2d69d..63d2eb0f 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.9.0" +version = "0.9.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 4fe2503c..69a69f51 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -11,33 +11,9 @@ pub struct VersionFile { pub size: u64, } -pub trait MinimumFileObject: AsyncRead + Send {} -impl MinimumFileObject for T {} +pub trait MinimumFileObject: AsyncRead + Send + Unpin {} +impl MinimumFileObject for T {} -// Intentionally not a generic, because of types in read_file -pub struct ReadToAsyncRead<'a> { - pub inner: Box, -} - -const ASYNC_READ_BUFFER_SIZE: usize = 8128; - -impl<'a> AsyncRead for ReadToAsyncRead<'a> { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; - let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); - match self.inner.read(&mut read_buf[0..read_size]) { - Ok(read) => { - buf.put_slice(&read_buf[0..read]); - std::task::Poll::Ready(Ok(())) - } - Err(err) => std::task::Poll::Ready(Err(err)), - } - } -} #[async_trait] pub trait VersionBackend: DynClone { From 4dfb02b0e7b83dee2fd500633aad6bd6df344a03 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 14:35:16 +1100 Subject: [PATCH 16/50] fix: remove lifetimes --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/versions/backends.rs | 4 ++-- libraries/droplet/src/versions/types.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 2db3ae35..d0f55ea4 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -175,7 +175,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 63d2eb0f..07e1dadc 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.9.1" +version = "0.9.2" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index c42eb80a..cf1e408d 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -69,7 +69,7 @@ impl VersionBackend for PathVersionBackend { file: &VersionFile, start: u64, end: u64, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; if start != 0 { @@ -197,7 +197,7 @@ impl VersionBackend for ZipVersionBackend { file: &VersionFile, start: u64, end: u64, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let mut read_command = Command::new("7z"); read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); let mut output = read_command diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 69a69f51..1f824e44 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -25,7 +25,7 @@ pub trait VersionBackend: DynClone { file: &VersionFile, start: u64, end: u64, - ) -> anyhow::Result>; + ) -> anyhow::Result>; } dyn_clone::clone_trait_object!(VersionBackend); From a3ee266f9d5e72daa2e45a82227a1192d55f9730 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 13 Dec 2025 15:27:03 +1100 Subject: [PATCH 17/50] feat: manifest generator --- libraries/droplet/Cargo.lock | 249 +++++++++++++++++++++++++++++- libraries/droplet/Cargo.toml | 5 + libraries/droplet/src/lib.rs | 1 + libraries/droplet/src/manifest.rs | 193 +++++++++++++++++++++++ 4 files changed, 442 insertions(+), 6 deletions(-) create mode 100644 libraries/droplet/src/manifest.rs diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index d0f55ea4..2845c587 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -98,6 +98,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + [[package]] name = "bytes" version = "1.11.0" @@ -119,6 +134,25 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "data-encoding" version = "2.8.0" @@ -162,6 +196,16 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -181,11 +225,16 @@ dependencies = [ "async-trait", "dyn-clone", "hex", + "humansize", "rcgen", "ring", + "serde", + "serde_json", + "sha2", "time", "time-macros", "tokio", + "uuid", "webpki", "x509-parser 0.17.0", ] @@ -196,6 +245,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -207,18 +266,49 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -231,6 +321,12 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "memchr" version = "2.7.4" @@ -316,6 +412,12 @@ dependencies = [ "asn1-rs 0.7.1", ] +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + [[package]] name = "pem" version = "3.0.5" @@ -356,6 +458,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rcgen" version = "0.13.2" @@ -378,7 +486,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "untrusted", "windows-sys 0.52.0", @@ -400,25 +508,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] -name = "serde" -version = "1.0.219" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -541,6 +695,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -553,12 +713,83 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki" version = "0.22.4" @@ -657,6 +888,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "x509-parser" version = "0.16.0" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 07e1dadc..30442436 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -16,6 +16,11 @@ dyn-clone = "1.0.20" tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] } anyhow = "1.0.100" async-trait = "0.1.89" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.145" +humansize = "2.1.3" +uuid = { version = "1.19.0", features = ["v4"] } +sha2 = "0.10.9" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index ec1ad139..99364c0d 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -3,3 +3,4 @@ pub mod file_utils; pub mod ssl; pub mod versions; +pub mod manifest; \ No newline at end of file diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs new file mode 100644 index 00000000..57d952c1 --- /dev/null +++ b/libraries/droplet/src/manifest.rs @@ -0,0 +1,193 @@ +use std::{collections::HashMap, path::Path}; + +use anyhow::anyhow; +use hex::ToHex as _; +use humansize::{BINARY, format_size}; +use serde::Serialize; +use serde_json::json; +use sha2::{Digest as _, Sha256}; +use tokio::io::AsyncReadExt as _; + +#[derive(Serialize)] +struct FileEntry { + filename: String, + start: usize, + length: usize, + permissions: u32, +} + +#[derive(Serialize)] +struct ChunkData { + files: Vec, + checksum: String, +} + +#[derive(Serialize)] +struct Manifest { + version: String, + chunks: HashMap, + size: u64, +} + +const CHUNK_SIZE: u64 = 1024 * 1024 * 64; +const WIGGLE: u64 = 1024 * 1024 * 1; + +use crate::versions::{create_backend_constructor, types::VersionFile}; + +pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( + dir: &Path, + progress_sfn: V, + log_sfn: T, +) -> anyhow::Result { + let mut backend = + create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; + + let required_single_file = true; //backend.require_whole_files(); + + let files = backend.list_files().await?; + // Filepath to chunk data + let mut chunks: Vec> = Vec::new(); + let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); + + log_sfn(format!("organizing files into chunks...",)); + + for version_file in files { + // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on + if required_single_file && version_file.size >= CHUNK_SIZE { + let size = version_file.size; + chunks.push(vec![(version_file, 0, size)]); + + continue; + } + + let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::(); + + // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be + if required_single_file { + let size = version_file.size.try_into().unwrap(); + current_chunk.push((version_file, 0, size)); + + current_size += size; + + if current_size >= CHUNK_SIZE { + // Pop current and add, then reset + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + } + + continue; + } + + // Otherwise we calculate how much of the file we need, then use that much + let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size; + if version_file.size >= remaining_budget { + let remaining_budget = CHUNK_SIZE - current_size; + current_chunk.push((version_file.clone(), 0, remaining_budget)); + + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + + let remaining_size = version_file.size - remaining_budget; + let mut running_offset = remaining_budget; + // Do everything but the last one + while running_offset < remaining_size { + let chunk_size = CHUNK_SIZE.min(remaining_size); + let chunk = vec![(version_file.clone(), running_offset, chunk_size)]; + if chunk_size == CHUNK_SIZE { + chunks.push(chunk); + } else { + current_chunk = chunk; + } + running_offset += chunk_size; + } + + continue; + } else { + let size = version_file.size; + current_chunk.push((version_file, 0, size)); + current_size += size; + } + + if current_size >= CHUNK_SIZE { + // Pop current and add, then reset + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + } + } + if current_chunk.len() > 0 { + chunks.push(current_chunk); + } + + log_sfn(format!( + "organized into {} chunks, generating checksums...", + chunks.len() + )); + + let mut manifest: HashMap = HashMap::new(); + let mut total_manifest_length = 0; + + let mut read_buf = vec![0; 1024 * 1024 * 64]; + + let chunk_len = chunks.len(); + for (index, chunk) in chunks.into_iter().enumerate() { + let uuid = uuid::Uuid::new_v4().to_string(); + let mut hasher = Sha256::new(); + + let mut chunk_data = ChunkData { + files: Vec::new(), + checksum: String::new(), + }; + + let mut chunk_length = 0; + + for (file, start, length) in chunk { + log_sfn(format!( + "reading {} from {} to {}, {}", + file.relative_filename, + start, + start + length, + format_size(length, BINARY) + )); + let mut reader = backend.reader(&file, start, start + length).await?; + + loop { + let amount = reader.read(&mut read_buf).await?; + if amount == 0 { + break; + } + hasher.update(&read_buf[0..amount]); + } + + chunk_length += length; + + chunk_data.files.push(FileEntry { + filename: file.relative_filename, + start: start.try_into().unwrap(), + length: length.try_into().unwrap(), + permissions: file.permission, + }); + } + + log_sfn(format!( + "created chunk of size {} ({}/{})", + format_size(chunk_length, BINARY), + index, + chunk_len + )); + total_manifest_length += chunk_length; + + let hash: String = hasher.finalize().encode_hex(); + chunk_data.checksum = hash; + manifest.insert(uuid, chunk_data); + + let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32; + progress_sfn(progress); + } + + Ok(json!(Manifest { + version: "2".to_string(), + chunks: manifest, + size: total_manifest_length + }) + .to_string()) +} From bef9f8d25dc6e34813af2b79bf5794c0714da284 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 13 Dec 2025 15:28:04 +1100 Subject: [PATCH 18/50] bump version --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 2845c587..fbd6236a 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.9.2" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 30442436..1c70994f 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.9.2" +version = "0.10.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" From 62ff8be3c0fa696b71a3fced0887412a5f5cadbc Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 13 Dec 2025 21:14:59 +1100 Subject: [PATCH 19/50] feat: multithreaded manifest generation --- libraries/droplet/Cargo.lock | 116 ++++++++++++++- libraries/droplet/Cargo.toml | 5 +- libraries/droplet/src/lib.rs | 1 + libraries/droplet/src/main.rs | 19 +++ libraries/droplet/src/manifest.rs | 159 +++++++++++++-------- libraries/droplet/src/versions/backends.rs | 4 +- 6 files changed, 242 insertions(+), 62 deletions(-) create mode 100644 libraries/droplet/src/main.rs diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index fbd6236a..3740dca1 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,11 +219,12 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.10.0" +version = "0.11.0" dependencies = [ "anyhow", "async-trait", "dyn-clone", + "futures", "hex", "humansize", "rcgen", @@ -245,6 +246,95 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -434,6 +524,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "powerfmt" version = "0.2.0" @@ -588,6 +684,12 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "syn" version = "2.0.100" @@ -692,9 +794,21 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typenum" version = "1.19.0" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 1c70994f..5447a91d 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.10.0" +version = "0.11.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -13,7 +13,7 @@ time = "0.3.41" webpki = "0.22.4" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] } +tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "sync", "macros", "rt-multi-thread"] } anyhow = "1.0.100" async-trait = "0.1.89" serde = { version = "1.0.228", features = ["derive"] } @@ -21,6 +21,7 @@ serde_json = "1.0.145" humansize = "2.1.3" uuid = { version = "1.19.0", features = ["v4"] } sha2 = "0.10.9" +futures = "0.3.31" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 99364c0d..4a873503 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -1,4 +1,5 @@ #![deny(clippy::all)] +#![feature(impl_trait_in_bindings)] pub mod file_utils; pub mod ssl; diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs new file mode 100644 index 00000000..724618b9 --- /dev/null +++ b/libraries/droplet/src/main.rs @@ -0,0 +1,19 @@ +use std::path::PathBuf; + +use droplet_rs::manifest::generate_manifest_rusty; +use tokio::runtime::{Handle, Runtime}; + +#[tokio::main] +pub async fn main() { + let metrics = Handle::current().metrics(); + println!("using {} workers", metrics.num_workers()); + generate_manifest_rusty( + &PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/Savage Resurrection"), + |_| {}, + |message| { + println!("{}", message); + }, + ) + .await + .unwrap(); +} diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 57d952c1..ba759d25 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -1,14 +1,23 @@ -use std::{collections::HashMap, path::Path}; +use std::{ + collections::HashMap, + future::Future, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; +use futures::{stream::FuturesUnordered, StreamExt}; use hex::ToHex as _; -use humansize::{BINARY, format_size}; +use humansize::{format_size, BINARY}; use serde::Serialize; use serde_json::json; use sha2::{Digest as _, Sha256}; -use tokio::io::AsyncReadExt as _; +use tokio::{io::AsyncReadExt as _, join, sync::Mutex}; -#[derive(Serialize)] +#[derive(Serialize, Clone)] struct FileEntry { filename: String, start: usize, @@ -16,7 +25,7 @@ struct FileEntry { permissions: u32, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] struct ChunkData { files: Vec, checksum: String, @@ -24,9 +33,9 @@ struct ChunkData { #[derive(Serialize)] struct Manifest { - version: String, - chunks: HashMap, - size: u64, + version: String, + chunks: HashMap, + size: u64, } const CHUNK_SIZE: u64 = 1024 * 1024 * 64; @@ -42,7 +51,7 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let mut backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; - let required_single_file = true; //backend.require_whole_files(); + let required_single_file = backend.require_whole_files(); let files = backend.list_files().await?; // Filepath to chunk data @@ -123,71 +132,107 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( chunks.len() )); - let mut manifest: HashMap = HashMap::new(); - let mut total_manifest_length = 0; + let manifest: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let total_manifest_length = Arc::new(AtomicU64::new(0)); - let mut read_buf = vec![0; 1024 * 1024 * 64]; + let backend = Arc::new(Mutex::new(backend)); - let chunk_len = chunks.len(); + let futures: FuturesUnordered>> = + FuturesUnordered::new(); + let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); for (index, chunk) in chunks.into_iter().enumerate() { - let uuid = uuid::Uuid::new_v4().to_string(); - let mut hasher = Sha256::new(); + let send_log = send_log.clone(); + let backend = backend.clone(); + let total_manifest_length = total_manifest_length.clone(); + let manifest = manifest.clone(); + futures.push((async move || -> Result<(), Error> { + let mut read_buf = vec![0; 1024 * 1024 * 64]; - let mut chunk_data = ChunkData { - files: Vec::new(), - checksum: String::new(), - }; + let uuid = uuid::Uuid::new_v4().to_string(); + let mut hasher = Sha256::new(); - let mut chunk_length = 0; + let mut chunk_data = ChunkData { + files: Vec::new(), + checksum: String::new(), + }; - for (file, start, length) in chunk { - log_sfn(format!( - "reading {} from {} to {}, {}", - file.relative_filename, - start, - start + length, - format_size(length, BINARY) - )); - let mut reader = backend.reader(&file, start, start + length).await?; + let mut chunk_length = 0; - loop { - let amount = reader.read(&mut read_buf).await?; - if amount == 0 { - break; + for (file, start, length) in chunk { + /* + send_log + .send(format!( + "reading {} from {} to {}, {}", + file.relative_filename, + start, + start + length, + format_size(length, BINARY) + )) + .await?; + */ + let mut reader = { + let mut backend_lock = backend.lock().await; + let reader = backend_lock.reader(&file, start, start + length).await?; + reader + }; + + loop { + let amount = reader.read(&mut read_buf).await?; + if amount == 0 { + break; + } + hasher.update(&read_buf[0..amount]); } - hasher.update(&read_buf[0..amount]); + + chunk_length += length; + + chunk_data.files.push(FileEntry { + filename: file.relative_filename, + start: start.try_into().unwrap(), + length: length.try_into().unwrap(), + permissions: file.permission, + }); } - chunk_length += length; + send_log + .send(format!( + "created chunk of size {} (index {})", + format_size(chunk_length, BINARY), + index + )) + .await?; - chunk_data.files.push(FileEntry { - filename: file.relative_filename, - start: start.try_into().unwrap(), - length: length.try_into().unwrap(), - permissions: file.permission, - }); - } + total_manifest_length.fetch_add(chunk_length, Ordering::Relaxed); - log_sfn(format!( - "created chunk of size {} ({}/{})", - format_size(chunk_length, BINARY), - index, - chunk_len - )); - total_manifest_length += chunk_length; + let hash: String = hasher.finalize().encode_hex(); + chunk_data.checksum = hash; + { + let mut manifest_lock = manifest.lock().await; + manifest_lock.insert(uuid, chunk_data); + }; - let hash: String = hasher.finalize().encode_hex(); - chunk_data.checksum = hash; - manifest.insert(uuid, chunk_data); - - let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32; - progress_sfn(progress); + Ok(()) + })()); } + drop(send_log); + join!( + async move { + while let Some(message) = recieve_log.recv().await { + log_sfn(message); + } + }, + futures.collect::>>() + ); + + let manifest = manifest.lock().await; + let manifest = manifest.clone(); + let manifest_size = size_of_val(&manifest); + println!("manifest uses {} bytes", manifest_size); Ok(json!(Manifest { version: "2".to_string(), chunks: manifest, - size: total_manifest_length + size: total_manifest_length.fetch_add(0, Ordering::Relaxed) }) .to_string()) } diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index cf1e408d..fa469db8 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -195,8 +195,8 @@ impl VersionBackend for ZipVersionBackend { async fn reader( &mut self, file: &VersionFile, - start: u64, - end: u64, + _start: u64, + _end: u64, ) -> anyhow::Result> { let mut read_command = Command::new("7z"); read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); From ed0b7bcf421a40b6ebcaf21d6005ba3bd6b52a5b Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 13 Dec 2025 21:20:57 +1100 Subject: [PATCH 20/50] fix: progress reporting --- libraries/droplet/Cargo.lock | 12 ------- libraries/droplet/Cargo.toml | 2 -- libraries/droplet/src/main.rs | 6 ++-- libraries/droplet/src/manifest.rs | 41 +++++++++------------- libraries/droplet/src/versions/backends.rs | 7 ++-- libraries/droplet/src/versions/mod.rs | 3 +- libraries/droplet/src/versions/types.rs | 4 +-- 7 files changed, 27 insertions(+), 48 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 3740dca1..bddbc4ac 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -233,10 +233,8 @@ dependencies = [ "serde_json", "sha2", "time", - "time-macros", "tokio", "uuid", - "webpki", "x509-parser 0.17.0", ] @@ -904,16 +902,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "webpki" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "windows-link" version = "0.2.1" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 5447a91d..c1d0956c 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -8,9 +8,7 @@ description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performa [dependencies] hex = "0.4.3" -time-macros = "0.2.22" time = "0.3.41" -webpki = "0.22.4" ring = "0.17.14" dyn-clone = "1.0.20" tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "sync", "macros", "rt-multi-thread"] } diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index 724618b9..d1ae9efc 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use droplet_rs::manifest::generate_manifest_rusty; -use tokio::runtime::{Handle, Runtime}; +use tokio::runtime::Handle; #[tokio::main] pub async fn main() { @@ -9,7 +9,9 @@ pub async fn main() { println!("using {} workers", metrics.num_workers()); generate_manifest_rusty( &PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/Savage Resurrection"), - |_| {}, + |progress| { + println!("PROGRESS: {}", progress) + }, |message| { println!("{}", message); }, diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index ba759d25..44ff235d 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -39,11 +39,11 @@ struct Manifest { } const CHUNK_SIZE: u64 = 1024 * 1024 * 64; -const WIGGLE: u64 = 1024 * 1024 * 1; +const WIGGLE: u64 = 1024 * 1024; use crate::versions::{create_backend_constructor, types::VersionFile}; -pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( +pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, @@ -58,7 +58,7 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let mut chunks: Vec> = Vec::new(); let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); - log_sfn(format!("organizing files into chunks...",)); + log_sfn("organizing files into chunks...".to_string()); for version_file in files { // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on @@ -73,14 +73,14 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be if required_single_file { - let size = version_file.size.try_into().unwrap(); + let size = version_file.size; current_chunk.push((version_file, 0, size)); current_size += size; if current_size >= CHUNK_SIZE { // Pop current and add, then reset - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + let new_chunk = std::mem::take(&mut current_chunk); chunks.push(new_chunk); } @@ -93,7 +93,7 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let remaining_budget = CHUNK_SIZE - current_size; current_chunk.push((version_file.clone(), 0, remaining_budget)); - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + let new_chunk = std::mem::take(&mut current_chunk); chunks.push(new_chunk); let remaining_size = version_file.size - remaining_budget; @@ -119,11 +119,11 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( if current_size >= CHUNK_SIZE { // Pop current and add, then reset - let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + let new_chunk = std::mem::take(&mut current_chunk); chunks.push(new_chunk); } } - if current_chunk.len() > 0 { + if !current_chunk.is_empty() { chunks.push(current_chunk); } @@ -140,12 +140,13 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let futures: FuturesUnordered>> = FuturesUnordered::new(); let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); + let chunks_length = chunks.len(); for (index, chunk) in chunks.into_iter().enumerate() { let send_log = send_log.clone(); let backend = backend.clone(); let total_manifest_length = total_manifest_length.clone(); let manifest = manifest.clone(); - futures.push((async move || -> Result<(), Error> { + futures.push(async move { let mut read_buf = vec![0; 1024 * 1024 * 64]; let uuid = uuid::Uuid::new_v4().to_string(); @@ -159,17 +160,6 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let mut chunk_length = 0; for (file, start, length) in chunk { - /* - send_log - .send(format!( - "reading {} from {} to {}, {}", - file.relative_filename, - start, - start + length, - format_size(length, BINARY) - )) - .await?; - */ let mut reader = { let mut backend_lock = backend.lock().await; let reader = backend_lock.reader(&file, start, start + length).await?; @@ -196,8 +186,9 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( send_log .send(format!( - "created chunk of size {} (index {})", + "created chunk of size {} from {} files (index {})", format_size(chunk_length, BINARY), + chunk_data.files.len(), index )) .await?; @@ -212,13 +203,17 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( }; Ok(()) - })()); + }); } drop(send_log); join!( async move { + let mut current_progress = 0f32; + let total_progress = chunks_length as f32; while let Some(message) = recieve_log.recv().await { log_sfn(message); + current_progress += 1.0f32; + progress_sfn((current_progress / total_progress) * 100.0f32) } }, futures.collect::>>() @@ -226,8 +221,6 @@ pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( let manifest = manifest.lock().await; let manifest = manifest.clone(); - let manifest_size = size_of_val(&manifest); - println!("manifest uses {} bytes", manifest_size); Ok(json!(Manifest { version: "2".to_string(), diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index fa469db8..1eb1b825 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -159,8 +159,8 @@ impl VersionBackend for ZipVersionBackend { let raw_result = String::from_utf8(result.stdout)?; let files = raw_result .split("\n") - .filter(|v| v.len() > 0) - .map(|v| v.split(" ").filter(|v| v.len() > 0)); + .filter(|v| !v.is_empty()) + .map(|v| v.split(" ").filter(|v| !v.is_empty())); let mut results = Vec::new(); for file in files { @@ -179,8 +179,7 @@ impl VersionBackend for ZipVersionBackend { } results.push(VersionFile { relative_filename: name - .into_iter() - .map(|v| *v) + .into_iter().copied() .fold(String::new(), |a, b| a + b + " ") .trim_end() .to_owned(), diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 19e001a9..4127d9d2 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -41,8 +41,7 @@ pub fn create_backend_constructor<'a>( if let Some(extension) = path.extension().and_then(|v| v.to_str()) { let supported = SUPPORTED_FILE_EXTENSIONS .iter() - .find(|v| ***v == *extension) - .is_some(); + .any(|v| **v == *extension); if supported { let buf = path.to_path_buf(); return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 1f824e44..5a7cfd95 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -1,8 +1,8 @@ -use std::{fmt::Debug, io::Read}; +use std::fmt::Debug; use async_trait::async_trait; use dyn_clone::DynClone; -use tokio::io::{self, AsyncRead}; +use tokio::io::AsyncRead; #[derive(Debug, Clone)] pub struct VersionFile { From e37e3f620b605b0ee9a922dfd77f2401611faf5b Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 13 Dec 2025 21:28:20 +1100 Subject: [PATCH 21/50] feat: provide rusty manifest --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index bddbc4ac..99753d69 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.11.0" +version = "0.11.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index c1d0956c..f2141354 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.11.0" +version = "0.11.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 44ff235d..47e515c4 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -18,7 +18,7 @@ use sha2::{Digest as _, Sha256}; use tokio::{io::AsyncReadExt as _, join, sync::Mutex}; #[derive(Serialize, Clone)] -struct FileEntry { +pub struct FileEntry { filename: String, start: usize, length: usize, @@ -26,13 +26,13 @@ struct FileEntry { } #[derive(Serialize, Clone)] -struct ChunkData { +pub struct ChunkData { files: Vec, checksum: String, } #[derive(Serialize)] -struct Manifest { +pub struct Manifest { version: String, chunks: HashMap, size: u64, @@ -47,7 +47,7 @@ pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, -) -> anyhow::Result { +) -> anyhow::Result { let mut backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; @@ -222,10 +222,9 @@ pub async fn generate_manifest_rusty( let manifest = manifest.lock().await; let manifest = manifest.clone(); - Ok(json!(Manifest { + Ok(Manifest { version: "2".to_string(), chunks: manifest, size: total_manifest_length.fetch_add(0, Ordering::Relaxed) }) - .to_string()) } From d7f9faf602a5cae66acb9a39114a55199532c15d Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 18 Dec 2025 17:36:47 +1100 Subject: [PATCH 22/50] feat: new 7z output parser --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 4 +- libraries/droplet/src/lib.rs | 5 +- libraries/droplet/src/manifest.rs | 2 + libraries/droplet/src/tests.rs | 11 ++++ libraries/droplet/src/versions/backends.rs | 62 ++++++++++++---------- libraries/droplet/test.txt | 1 + 7 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 libraries/droplet/src/tests.rs create mode 100644 libraries/droplet/test.txt diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 99753d69..a5da8f87 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.11.1" +version = "0.11.2" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index f2141354..76f06cc4 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.11.1" +version = "0.11.2" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -11,7 +11,7 @@ hex = "0.4.3" time = "0.3.41" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "sync", "macros", "rt-multi-thread"] } +tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "rt", "rt-multi-thread", "macros", "sync"] } anyhow = "1.0.100" async-trait = "0.1.89" serde = { version = "1.0.228", features = ["derive"] } diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 4a873503..6781cc7a 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -4,4 +4,7 @@ pub mod file_utils; pub mod ssl; pub mod versions; -pub mod manifest; \ No newline at end of file +pub mod manifest; + +#[cfg(test)] +pub mod tests; diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 47e515c4..7b0b015a 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -29,6 +29,7 @@ pub struct FileEntry { pub struct ChunkData { files: Vec, checksum: String, + //iv: [u8; 16] } #[derive(Serialize)] @@ -155,6 +156,7 @@ pub async fn generate_manifest_rusty( let mut chunk_data = ChunkData { files: Vec::new(), checksum: String::new(), + //iv: }; let mut chunk_length = 0; diff --git a/libraries/droplet/src/tests.rs b/libraries/droplet/src/tests.rs new file mode 100644 index 00000000..99edf35b --- /dev/null +++ b/libraries/droplet/src/tests.rs @@ -0,0 +1,11 @@ +use std::path::PathBuf; + +use crate::versions::create_backend_constructor; + +#[tokio::test] +pub async fn test_7z_list() { + let zip_path = "/home/decduck/Dev/droplet/assets/TheGame.zip"; + let mut backend = create_backend_constructor(&PathBuf::from(zip_path)).unwrap()().unwrap(); + let files = backend.list_files().await.unwrap(); + tokio::fs::write("./test.txt", format!("{:?}", files)).await.unwrap(); +} diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index 1eb1b825..6ca9c23c 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -15,6 +15,7 @@ use tokio::{ io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, process::Command, }; +use x509_parser::nom::InputIter; use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; @@ -148,7 +149,7 @@ impl ZipVersionBackend { impl VersionBackend for ZipVersionBackend { async fn list_files(&mut self) -> anyhow::Result> { let mut list_command = Command::new("7z"); - list_command.args(vec!["l", "-ba", &self.path]); + list_command.args(vec!["l", &self.path]); let result = list_command.output().await?; if !result.status.success() { return Err(anyhow!( @@ -157,35 +158,38 @@ impl VersionBackend for ZipVersionBackend { )); } let raw_result = String::from_utf8(result.stdout)?; - let files = raw_result - .split("\n") - .filter(|v| !v.is_empty()) - .map(|v| v.split(" ").filter(|v| !v.is_empty())); - let mut results = Vec::new(); + let mut lines = raw_result.split("\n").skip(11); - for file in files { - let values = file.collect::>(); - let mut iter = values.iter(); - let (date, time, attrs, size, compress, name) = ( - iter.next().expect("failed to read date"), - iter.next().expect("failed to read time"), - iter.next().expect("failed to read attrs"), - iter.next().expect("failed to read size"), - iter.next().expect("failed to read compress"), - iter.collect::>(), - ); - if attrs.starts_with("D") { - continue; - } - results.push(VersionFile { - relative_filename: name - .into_iter().copied() - .fold(String::new(), |a, b| a + b + " ") - .trim_end() - .to_owned(), - permission: 0o744, // owner r/w/x, everyone else, read - size: size.parse().unwrap(), - }); + let mut column_lines = lines + .find(|v| v.starts_with('-')) + .ok_or(anyhow!("invalid 7z output"))? + .chars(); + let file_lines = lines.take_while(|v| !v.starts_with("-")); + + /* + Date Time Attr Size Compressed Name + ------------------- ----- ------------ ------------ ------------------------ + */ + let datetime_pos = 0usize; + let attrs_pos = datetime_pos + column_lines.position(|v| v == ' ').unwrap() + 1; + let size_pos = attrs_pos + column_lines.position(|v| v == ' ').unwrap() + 1; + let compressed_size_pos = size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; + let name_pos = compressed_size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; + + let mut results = Vec::new(); + for file in file_lines { + let name = file[name_pos..].trim(); + let size = file[size_pos..compressed_size_pos].trim(); + + let size = str::parse::(size)?; + + let version_file = VersionFile { + relative_filename: name.to_string(), + permission: 0o744, + size: size, + }; + + results.push(version_file); } Ok(results) diff --git a/libraries/droplet/test.txt b/libraries/droplet/test.txt new file mode 100644 index 00000000..f250dadf --- /dev/null +++ b/libraries/droplet/test.txt @@ -0,0 +1 @@ +[VersionFile { relative_filename: "setup.exe", permission: 484, size: 1024000000 }, VersionFile { relative_filename: "test file.txt", permission: 484, size: 12 }] \ No newline at end of file From 5c6597eb6b72acbe21067254484655270b77f7d9 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 18 Dec 2025 19:04:56 +1100 Subject: [PATCH 23/50] feat: iv and key creation --- libraries/droplet/Cargo.lock | 3 ++- libraries/droplet/Cargo.toml | 3 ++- libraries/droplet/src/main.rs | 4 +++- libraries/droplet/src/manifest.rs | 13 ++++++++++--- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index a5da8f87..beef5a7c 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,12 +219,13 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.11.2" +version = "0.12.0" dependencies = [ "anyhow", "async-trait", "dyn-clone", "futures", + "getrandom 0.3.4", "hex", "humansize", "rcgen", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 76f06cc4..c4cb1908 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.11.2" +version = "0.12.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -20,6 +20,7 @@ humansize = "2.1.3" uuid = { version = "1.19.0", features = ["v4"] } sha2 = "0.10.9" futures = "0.3.31" +getrandom = "0.3.4" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index d1ae9efc..d098a419 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -1,13 +1,14 @@ use std::path::PathBuf; use droplet_rs::manifest::generate_manifest_rusty; +use serde_json::json; use tokio::runtime::Handle; #[tokio::main] pub async fn main() { let metrics = Handle::current().metrics(); println!("using {} workers", metrics.num_workers()); - generate_manifest_rusty( + let manifest = generate_manifest_rusty( &PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/Savage Resurrection"), |progress| { println!("PROGRESS: {}", progress) @@ -18,4 +19,5 @@ pub async fn main() { ) .await .unwrap(); + tokio::fs::write("./manifst.json", json!(manifest).to_string()).await.expect("failed to write manifest"); } diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 7b0b015a..f67fd8eb 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -29,7 +29,7 @@ pub struct FileEntry { pub struct ChunkData { files: Vec, checksum: String, - //iv: [u8; 16] + iv: [u8; 16], } #[derive(Serialize)] @@ -37,6 +37,7 @@ pub struct Manifest { version: String, chunks: HashMap, size: u64, + key: [u8; 16], } const CHUNK_SIZE: u64 = 1024 * 1024 * 64; @@ -153,10 +154,12 @@ pub async fn generate_manifest_rusty( let uuid = uuid::Uuid::new_v4().to_string(); let mut hasher = Sha256::new(); + let mut iv = [0u8; 16]; + getrandom::fill(&mut iv).map_err(|err| anyhow!("failed to generate IV: {:?}", err))?; let mut chunk_data = ChunkData { files: Vec::new(), checksum: String::new(), - //iv: + iv, }; let mut chunk_length = 0; @@ -224,9 +227,13 @@ pub async fn generate_manifest_rusty( let manifest = manifest.lock().await; let manifest = manifest.clone(); + let mut key = [0u8; 16]; + getrandom::fill(&mut key).map_err(|err| anyhow!("failed to generate key: {:?}", err))?; + Ok(Manifest { version: "2".to_string(), chunks: manifest, - size: total_manifest_length.fetch_add(0, Ordering::Relaxed) + size: total_manifest_length.fetch_add(0, Ordering::Relaxed), + key, }) } From 8f24d4ea7510ca1e84b5cb5add39d42335fba04a Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 18 Dec 2025 19:39:03 +1100 Subject: [PATCH 24/50] fix: add deserialize traits for manifest --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index beef5a7c..833d6fd3 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.0" +version = "0.12.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index c4cb1908..1225bafe 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.12.0" +version = "0.12.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index f67fd8eb..a0efb6c7 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -12,12 +12,12 @@ use anyhow::{anyhow, Error}; use futures::{stream::FuturesUnordered, StreamExt}; use hex::ToHex as _; use humansize::{format_size, BINARY}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest as _, Sha256}; use tokio::{io::AsyncReadExt as _, join, sync::Mutex}; -#[derive(Serialize, Clone)] +#[derive(Serialize, Deserialize, Clone)] pub struct FileEntry { filename: String, start: usize, @@ -25,14 +25,14 @@ pub struct FileEntry { permissions: u32, } -#[derive(Serialize, Clone)] +#[derive(Serialize, Deserialize, Clone)] pub struct ChunkData { files: Vec, checksum: String, iv: [u8; 16], } -#[derive(Serialize)] +#[derive(Serialize, Deserialize)] pub struct Manifest { version: String, chunks: HashMap, From dac75556d33284967e7b32a88353e878c80ceecc Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 18 Dec 2025 19:49:37 +1100 Subject: [PATCH 25/50] fix: make manifest public --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 833d6fd3..8dbc9ad5 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.1" +version = "0.12.2" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 1225bafe..8627526a 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.12.1" +version = "0.12.2" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index a0efb6c7..4ee35e0c 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -19,25 +19,25 @@ use tokio::{io::AsyncReadExt as _, join, sync::Mutex}; #[derive(Serialize, Deserialize, Clone)] pub struct FileEntry { - filename: String, - start: usize, - length: usize, - permissions: u32, + pub filename: String, + pub start: usize, + pub length: usize, + pub permissions: u32, } #[derive(Serialize, Deserialize, Clone)] pub struct ChunkData { - files: Vec, - checksum: String, - iv: [u8; 16], + pub files: Vec, + pub checksum: String, + pub iv: [u8; 16], } #[derive(Serialize, Deserialize)] pub struct Manifest { - version: String, - chunks: HashMap, - size: u64, - key: [u8; 16], + pub version: String, + pub chunks: HashMap, + pub size: u64, + pub key: [u8; 16], } const CHUNK_SIZE: u64 = 1024 * 1024 * 64; From 56e6ecee26760ffce09633dd5a668118ef4711e7 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Fri, 19 Dec 2025 14:36:40 +1100 Subject: [PATCH 26/50] feat: clearer error messages --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/versions/backends.rs | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 8dbc9ad5..61bf6bab 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.2" +version = "0.12.3" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 8627526a..55503bbe 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.12.2" +version = "0.12.3" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index 6ca9c23c..3bef9dcd 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -15,7 +15,6 @@ use tokio::{ io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, process::Command, }; -use x509_parser::nom::InputIter; use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; @@ -55,7 +54,7 @@ impl VersionBackend for PathVersionBackend { self.peek_file( relative .to_str() - .ok_or(anyhow!("Could not parse path"))? + .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? .to_owned(), ) .await?, @@ -87,7 +86,7 @@ impl VersionBackend for PathVersionBackend { async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { let pathbuf = self.base_dir.join(sub_path.clone()); if !pathbuf.exists() { - return Err(anyhow!("Path doesn't exist.")); + return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy())); }; let file = File::open(pathbuf.clone()).await?; From 44efcbb633b7f269436d49d5531905ea607cbcbe Mon Sep 17 00:00:00 2001 From: DecDuck Date: Fri, 19 Dec 2025 14:50:45 +1100 Subject: [PATCH 27/50] fix: remove test.txt --- libraries/droplet/test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 libraries/droplet/test.txt diff --git a/libraries/droplet/test.txt b/libraries/droplet/test.txt deleted file mode 100644 index f250dadf..00000000 --- a/libraries/droplet/test.txt +++ /dev/null @@ -1 +0,0 @@ -[VersionFile { relative_filename: "setup.exe", permission: 484, size: 1024000000 }, VersionFile { relative_filename: "test file.txt", permission: 484, size: 12 }] \ No newline at end of file From 87642befed64c525df0596ff5915d07f18dfc504 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 20 Dec 2025 17:04:18 +1100 Subject: [PATCH 28/50] feat: fix droplet manifest generation --- libraries/droplet/.gitignore | 4 ++ libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 77 ++++++++++++++----------------- 4 files changed, 41 insertions(+), 44 deletions(-) diff --git a/libraries/droplet/.gitignore b/libraries/droplet/.gitignore index 423b4790..d80b04ff 100644 --- a/libraries/droplet/.gitignore +++ b/libraries/droplet/.gitignore @@ -19,3 +19,7 @@ target/ # Added by cargo /target + +perf.data +flamegraph.svg +*.json \ No newline at end of file diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 61bf6bab..803b4f4c 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.3" +version = "0.13.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 55503bbe..18ef4aab 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.12.3" +version = "0.13.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 4ee35e0c..29c56dd1 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -55,26 +55,25 @@ pub async fn generate_manifest_rusty( let required_single_file = backend.require_whole_files(); - let files = backend.list_files().await?; + let mut files = backend.list_files().await?; + files.sort_by(|a, b| a.size.cmp(&b.size)); // Filepath to chunk data let mut chunks: Vec> = Vec::new(); let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); log_sfn("organizing files into chunks...".to_string()); - for version_file in files { - // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on - if required_single_file && version_file.size >= CHUNK_SIZE { - let size = version_file.size; - chunks.push(vec![(version_file, 0, size)]); + if required_single_file { + for version_file in files { + if version_file.size >= CHUNK_SIZE { + let size = version_file.size; + chunks.push(vec![(version_file, 0, size)]); - continue; - } + continue; + } - let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::(); + let mut current_size = current_chunk.iter().map(|v| v.2).sum::(); - // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be - if required_single_file { let size = version_file.size; current_chunk.push((version_file, 0, size)); @@ -88,43 +87,36 @@ pub async fn generate_manifest_rusty( continue; } + } else { + for version_file in files { + let current_size = current_chunk.iter().map(|v| v.2).sum::(); - // Otherwise we calculate how much of the file we need, then use that much - let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size; - if version_file.size >= remaining_budget { - let remaining_budget = CHUNK_SIZE - current_size; - current_chunk.push((version_file.clone(), 0, remaining_budget)); + if version_file.size + current_size < CHUNK_SIZE + WIGGLE { + let size = version_file.size; + current_chunk.push((version_file, 0, size)); - let new_chunk = std::mem::take(&mut current_chunk); - chunks.push(new_chunk); - - let remaining_size = version_file.size - remaining_budget; - let mut running_offset = remaining_budget; - // Do everything but the last one - while running_offset < remaining_size { - let chunk_size = CHUNK_SIZE.min(remaining_size); - let chunk = vec![(version_file.clone(), running_offset, chunk_size)]; - if chunk_size == CHUNK_SIZE { - chunks.push(chunk); - } else { - current_chunk = chunk; - } - running_offset += chunk_size; + continue; } - continue; - } else { - let size = version_file.size; - current_chunk.push((version_file, 0, size)); - current_size += size; - } + // Fill up current chunk + let remaining = CHUNK_SIZE - current_size; + current_chunk.push((version_file.clone(), 0, remaining)); + chunks.push(std::mem::take(&mut current_chunk)); - if current_size >= CHUNK_SIZE { - // Pop current and add, then reset - let new_chunk = std::mem::take(&mut current_chunk); - chunks.push(new_chunk); + // This is our offset in our current file + let mut offset = remaining; + while offset < version_file.size { + let length = CHUNK_SIZE.min(version_file.size - offset); + offset += length; + if length == CHUNK_SIZE { + chunks.push(vec![(version_file.clone(), offset, length)]); + } else { + current_chunk.push((version_file.clone(), offset, length)); + } + } } } + if !current_chunk.is_empty() { chunks.push(current_chunk); } @@ -191,8 +183,9 @@ pub async fn generate_manifest_rusty( send_log .send(format!( - "created chunk of size {} from {} files (index {})", + "created chunk of size {} ({}b) from {} files (index {})", format_size(chunk_length, BINARY), + chunk_length, chunk_data.files.len(), index )) From 3ff2944f77fd4e2676231c71b9e7b2a910d1239f Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 20 Dec 2025 18:35:24 +1100 Subject: [PATCH 29/50] fix: manifest generation no actually this time --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/main.rs | 42 ++++++++++++++++++++++++++----- libraries/droplet/src/manifest.rs | 8 +++--- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 803b4f4c..e6dcc97a 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.13.0" +version = "0.14.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 18ef4aab..4e609c44 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.13.0" +version = "0.14.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index d098a419..76036a45 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{os::unix::fs::MetadataExt, path::PathBuf}; use droplet_rs::manifest::generate_manifest_rusty; use serde_json::json; @@ -6,18 +6,48 @@ use tokio::runtime::Handle; #[tokio::main] pub async fn main() { + let target_dir = + PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/BloonsTD6"); let metrics = Handle::current().metrics(); println!("using {} workers", metrics.num_workers()); let manifest = generate_manifest_rusty( - &PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/Savage Resurrection"), - |progress| { - println!("PROGRESS: {}", progress) - }, + &target_dir, + |progress| println!("PROGRESS: {}", progress), |message| { println!("{}", message); }, ) .await .unwrap(); - tokio::fs::write("./manifst.json", json!(manifest).to_string()).await.expect("failed to write manifest"); + + // Sanity checks + for (_, chunk_data) in manifest.chunks { + for file in chunk_data.files { + let path = target_dir.join(file.filename); + if !path.exists() { + panic!("{} doesn't exist", path.display()); + } + + let metadata = path.metadata().expect("failed to fetch metadata"); + let file_size = metadata.size(); + if file.start > file_size as usize { + panic!( + "start for {} doesn't make sense: start: {}, size: {}", + path.display(), + file.start, + file_size + ); + } + + let end_position = file.start + file.length; + if end_position > file_size as usize { + panic!( + "end for {} doesn't make sense: end: {}, size: {}", + path.display(), + end_position, + file_size + ); + } + } + } } diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 29c56dd1..7b66f306 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -56,7 +56,7 @@ pub async fn generate_manifest_rusty( let required_single_file = backend.require_whole_files(); let mut files = backend.list_files().await?; - files.sort_by(|a, b| a.size.cmp(&b.size)); + files.sort_by(|a, b| b.size.cmp(&a.size)); // Filepath to chunk data let mut chunks: Vec> = Vec::new(); let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); @@ -91,7 +91,7 @@ pub async fn generate_manifest_rusty( for version_file in files { let current_size = current_chunk.iter().map(|v| v.2).sum::(); - if version_file.size + current_size < CHUNK_SIZE + WIGGLE { + if version_file.size + current_size < CHUNK_SIZE { let size = version_file.size; current_chunk.push((version_file, 0, size)); @@ -101,18 +101,18 @@ pub async fn generate_manifest_rusty( // Fill up current chunk let remaining = CHUNK_SIZE - current_size; current_chunk.push((version_file.clone(), 0, remaining)); - chunks.push(std::mem::take(&mut current_chunk)); + chunks.push(std::mem::replace(&mut current_chunk, Vec::new())); // This is our offset in our current file let mut offset = remaining; while offset < version_file.size { let length = CHUNK_SIZE.min(version_file.size - offset); - offset += length; if length == CHUNK_SIZE { chunks.push(vec![(version_file.clone(), offset, length)]); } else { current_chunk.push((version_file.clone(), offset, length)); } + offset += length; } } } From a6cb297e605c2972f9dab069ccc0447fb7d1e2dc Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 20 Dec 2025 19:48:40 +1100 Subject: [PATCH 30/50] fix: limit chunks to 512 files each --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index e6dcc97a..7fff6478 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.14.0" +version = "0.14.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 4e609c44..104c576a 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.14.0" +version = "0.14.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 7b66f306..d8f38cde 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -41,7 +41,7 @@ pub struct Manifest { } const CHUNK_SIZE: u64 = 1024 * 1024 * 64; -const WIGGLE: u64 = 1024 * 1024; +const MAX_FILE_COUNT: usize = 512; use crate::versions::{create_backend_constructor, types::VersionFile}; @@ -85,10 +85,18 @@ pub async fn generate_manifest_rusty( chunks.push(new_chunk); } + if current_chunk.len() >= MAX_FILE_COUNT { + chunks.push(std::mem::take(&mut current_chunk)); + } + continue; } } else { for version_file in files { + if current_chunk.len() >= MAX_FILE_COUNT { + chunks.push(std::mem::take(&mut current_chunk)); + } + let current_size = current_chunk.iter().map(|v| v.2).sum::(); if version_file.size + current_size < CHUNK_SIZE { From 28a1152ed23a0177a494db06e6fdfff8bf60a9f8 Mon Sep 17 00:00:00 2001 From: quexeky Date: Thu, 22 Jan 2026 07:35:23 +1100 Subject: [PATCH 31/50] chore: Remove static target_dir from main.rs --- libraries/droplet/src/main.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index 76036a45..baff5528 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -1,15 +1,16 @@ -use std::{os::unix::fs::MetadataExt, path::PathBuf}; +use std::{env, os::unix::fs::MetadataExt, path::PathBuf}; use droplet_rs::manifest::generate_manifest_rusty; -use serde_json::json; use tokio::runtime::Handle; #[tokio::main] pub async fn main() { - let target_dir = - PathBuf::from("/home/decduck/.local/share/Steam/steamapps/common/BloonsTD6"); + let mut args = env::args(); + let target_dir = PathBuf::from(args.nth(1).expect("Provide target directory")); + let metrics = Handle::current().metrics(); println!("using {} workers", metrics.num_workers()); + let manifest = generate_manifest_rusty( &target_dir, |progress| println!("PROGRESS: {}", progress), From e7297019eaf12929a6d239077c47b93d23595f53 Mon Sep 17 00:00:00 2001 From: quexeky Date: Thu, 22 Jan 2026 07:42:58 +1100 Subject: [PATCH 32/50] chore: Add flake.nix and .envrc --- libraries/droplet/.envrc | 1 + libraries/droplet/.gitignore | 4 +- libraries/droplet/flake.lock | 96 ++++++++++++++++++++++++++++++++++++ libraries/droplet/flake.nix | 52 +++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 libraries/droplet/.envrc create mode 100644 libraries/droplet/flake.lock create mode 100644 libraries/droplet/flake.nix diff --git a/libraries/droplet/.envrc b/libraries/droplet/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/libraries/droplet/.envrc @@ -0,0 +1 @@ +use flake diff --git a/libraries/droplet/.gitignore b/libraries/droplet/.gitignore index d80b04ff..340991a0 100644 --- a/libraries/droplet/.gitignore +++ b/libraries/droplet/.gitignore @@ -22,4 +22,6 @@ target/ perf.data flamegraph.svg -*.json \ No newline at end of file +*.json + +.direnv \ No newline at end of file diff --git a/libraries/droplet/flake.lock b/libraries/droplet/flake.lock new file mode 100644 index 00000000..e0d2640d --- /dev/null +++ b/libraries/droplet/flake.lock @@ -0,0 +1,96 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1768886240, + "narHash": "sha256-C2TjvwYZ2VDxYWeqvvJ5XPPp6U7H66zeJlRaErJKoEM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "80e4adbcf8992d3fd27ad4964fbb84907f9478b0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1744536153, + "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1768963622, + "narHash": "sha256-n6VHiUgrYD9yjagzG6ncVVqFbVTsKCI54tR9PNAFCo0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "2ef5b3362af585a83bafd34e7fc9b1f388c2e5e2", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/libraries/droplet/flake.nix b/libraries/droplet/flake.nix new file mode 100644 index 00000000..7c806fd3 --- /dev/null +++ b/libraries/droplet/flake.nix @@ -0,0 +1,52 @@ +{ + description = "Drop-OSS app development environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + rust-overlay.url = "github:oxalica/rust-overlay"; + }; + + outputs = + { + self, + nixpkgs, + flake-utils, + rust-overlay, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + overlays = [ (import rust-overlay) ]; + pkgs = import nixpkgs { + inherit system overlays; + }; + libraries = with pkgs; [ + glib + glibc + openssl + ]; + in + { + devShells.default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + pkg-config + git + rust-bin.nightly.latest.default + rust-analyzer + cargo-expand + ]; + + + buildInputs = libraries; + + shellHook = '' + export LD_LIBRARY_PATH="${ + pkgs.lib.makeLibraryPath libraries + }:$LD_LIBRARY_PATH" + echo "Downpour development environment loaded" + ''; + }; + } + ); +} From 23491695b3a04fc4ac4ddab44e92da94e0a4ff36 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 4 Feb 2026 22:39:52 +1100 Subject: [PATCH 33/50] feat: optional reader semaphore --- libraries/droplet/src/main.rs | 1 + libraries/droplet/src/manifest.rs | 30 ++++++++++++++-------- libraries/droplet/src/versions/backends.rs | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index baff5528..9f163ab4 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -17,6 +17,7 @@ pub async fn main() { |message| { println!("{}", message); }, + &None, ) .await .unwrap(); diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index d8f38cde..9274ad2c 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -1,11 +1,7 @@ use std::{ - collections::HashMap, - future::Future, - path::Path, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, + collections::HashMap, future::Future, path::Path, sync::{ + Arc, atomic::{AtomicU64, Ordering} + } }; use anyhow::{anyhow, Error}; @@ -13,9 +9,12 @@ use futures::{stream::FuturesUnordered, StreamExt}; use hex::ToHex as _; use humansize::{format_size, BINARY}; use serde::{Deserialize, Serialize}; -use serde_json::json; use sha2::{Digest as _, Sha256}; -use tokio::{io::AsyncReadExt as _, join, sync::Mutex}; +use tokio::{ + io::AsyncReadExt as _, + join, + sync::{Mutex, Semaphore}, +}; #[derive(Serialize, Deserialize, Clone)] pub struct FileEntry { @@ -49,6 +48,7 @@ pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, + reader_semaphore: &Option, ) -> anyhow::Result { let mut backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; @@ -56,7 +56,7 @@ pub async fn generate_manifest_rusty( let required_single_file = backend.require_whole_files(); let mut files = backend.list_files().await?; - files.sort_by(|a, b| b.size.cmp(&a.size)); + files.sort_by_key(|b| std::cmp::Reverse(b.size)); // Filepath to chunk data let mut chunks: Vec> = Vec::new(); let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); @@ -109,7 +109,7 @@ pub async fn generate_manifest_rusty( // Fill up current chunk let remaining = CHUNK_SIZE - current_size; current_chunk.push((version_file.clone(), 0, remaining)); - chunks.push(std::mem::replace(&mut current_chunk, Vec::new())); + chunks.push(std::mem::take(&mut current_chunk)); // This is our offset in our current file let mut offset = remaining; @@ -165,6 +165,12 @@ pub async fn generate_manifest_rusty( let mut chunk_length = 0; for (file, start, length) in chunk { + let permit = if let Some(reader_semaphore) = &reader_semaphore { + Some(reader_semaphore.acquire().await?) + } else { + None + }; + let mut reader = { let mut backend_lock = backend.lock().await; let reader = backend_lock.reader(&file, start, start + length).await?; @@ -187,6 +193,8 @@ pub async fn generate_manifest_rusty( length: length.try_into().unwrap(), permissions: file.permission, }); + + drop(permit); } send_log diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index 3bef9dcd..ecd6876b 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -185,7 +185,7 @@ impl VersionBackend for ZipVersionBackend { let version_file = VersionFile { relative_filename: name.to_string(), permission: 0o744, - size: size, + size, }; results.push(version_file); From 047a1184f90c6752548b9c8588ce15b3fa9c94e7 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 4 Feb 2026 22:40:11 +1100 Subject: [PATCH 34/50] feat: bump version --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 7fff6478..a698ec9d 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.14.1" +version = "0.15.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 104c576a..dbd1461d 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.14.1" +version = "0.15.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" From dedf57517af36fad8768629d8440a1d3ef895acc Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 4 Feb 2026 22:46:03 +1100 Subject: [PATCH 35/50] fix: semaphore reference --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/main.rs | 2 +- libraries/droplet/src/manifest.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index a698ec9d..7f99250a 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.15.0" +version = "0.15.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index dbd1461d..27dab856 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.15.0" +version = "0.15.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index 9f163ab4..a9289652 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -17,7 +17,7 @@ pub async fn main() { |message| { println!("{}", message); }, - &None, + None, ) .await .unwrap(); diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 9274ad2c..8f7f9e50 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -48,7 +48,7 @@ pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, - reader_semaphore: &Option, + reader_semaphore: Option<&Semaphore>, ) -> anyhow::Result { let mut backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; From c8449d7b3ee05c6fcc19beaa58a70e1560b77a5b Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:18:39 +1100 Subject: [PATCH 36/50] feat: libarchive backend --- libraries/droplet/.gitmodules | 3 + libraries/droplet/Cargo.lock | 186 ++++++++++++-- libraries/droplet/Cargo.toml | 12 +- libraries/droplet/README.md | 8 +- libraries/droplet/libarchive-rust | 1 + libraries/droplet/src/lib.rs | 3 + libraries/droplet/src/main.rs | 2 + libraries/droplet/src/manifest.rs | 53 ++-- .../droplet/src/versions/archive_backend.rs | 115 +++++++++ libraries/droplet/src/versions/backends.rs | 230 ------------------ libraries/droplet/src/versions/mod.rs | 66 +++-- .../droplet/src/versions/path_backend.rs | 98 ++++++++ libraries/droplet/src/versions/types.rs | 11 +- libraries/droplet/src/vm/mod.rs | 0 14 files changed, 482 insertions(+), 306 deletions(-) create mode 100644 libraries/droplet/.gitmodules create mode 160000 libraries/droplet/libarchive-rust create mode 100644 libraries/droplet/src/versions/archive_backend.rs delete mode 100644 libraries/droplet/src/versions/backends.rs create mode 100644 libraries/droplet/src/versions/path_backend.rs create mode 100644 libraries/droplet/src/vm/mod.rs diff --git a/libraries/droplet/.gitmodules b/libraries/droplet/.gitmodules new file mode 100644 index 00000000..8dfab4fd --- /dev/null +++ b/libraries/droplet/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libarchive-rust"] + path = libarchive-rust + url = git@github.com:Drop-OSS/libarchive-rust.git diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 7f99250a..157e5988 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "anyhow" version = "1.0.100" @@ -48,8 +63,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.117", + "synstructure 0.13.1", ] [[package]] @@ -60,8 +75,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.117", + "synstructure 0.13.1", ] [[package]] @@ -72,7 +87,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -83,7 +98,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -92,6 +107,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -214,7 +244,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -228,11 +258,13 @@ dependencies = [ "getrandom 0.3.4", "hex", "humansize", + "libarchive", "rcgen", "ring", "serde", "serde_json", "sha2", + "speedometer", "time", "tokio", "uuid", @@ -245,6 +277,28 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "futures" version = "0.3.31" @@ -301,7 +355,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -367,6 +421,12 @@ dependencies = [ "wasip2", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "hex" version = "0.4.3" @@ -405,10 +465,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "libc" -version = "0.2.171" +name = "libarchive" +version = "0.1.1" +dependencies = [ + "libarchive3-sys", + "libc", +] + +[[package]] +name = "libarchive3-sys" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "libc" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libm" @@ -428,6 +506,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.1.0" @@ -483,6 +570,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -529,6 +625,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "powerfmt" version = "0.2.0" @@ -587,6 +689,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -641,7 +749,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -690,16 +798,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] -name = "syn" -version = "2.0.100" +name = "speedometer" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e" +dependencies = [ + "failure", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.1" @@ -708,7 +848,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -737,7 +877,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -748,7 +888,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -805,7 +945,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -820,6 +960,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -890,7 +1036,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 27dab856..eaaddaf9 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -11,7 +11,15 @@ hex = "0.4.3" time = "0.3.41" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "rt", "rt-multi-thread", "macros", "sync"] } +tokio = { version = "^1.48.0", features = [ + "process", + "fs", + "io-util", + "rt", + "rt-multi-thread", + "macros", + "sync", +] } anyhow = "1.0.100" async-trait = "0.1.89" serde = { version = "1.0.228", features = ["derive"] } @@ -21,6 +29,8 @@ uuid = { version = "1.19.0", features = ["v4"] } sha2 = "0.10.9" futures = "0.3.31" getrandom = "0.3.4" +libarchive = { version = "*", path = "./libarchive-rust" } +speedometer = "0.2.2" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/README.md b/libraries/droplet/README.md index b321a75e..266e5152 100644 --- a/libraries/droplet/README.md +++ b/libraries/droplet/README.md @@ -1,2 +1,8 @@ # droplet-rs -Core rust functionality for droplet + +A Rust-based library for utilities and functionality required both by the server, client, and other tools. + +## manifest generation +`droplet-rs` contains the manifest generation code, held in `manifest.rs`. + +It also includes the version backends to provide a unified read-write interface to files. \ No newline at end of file diff --git a/libraries/droplet/libarchive-rust b/libraries/droplet/libarchive-rust new file mode 160000 index 00000000..279fd5f7 --- /dev/null +++ b/libraries/droplet/libarchive-rust @@ -0,0 +1 @@ +Subproject commit 279fd5f727d5af50771cdfd660d657dff52c42f7 diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 6781cc7a..8ad90277 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -5,6 +5,9 @@ pub mod file_utils; pub mod ssl; pub mod versions; pub mod manifest; +pub mod vm; + +extern crate libarchive; #[cfg(test)] pub mod tests; diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index a9289652..c512fe89 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -22,6 +22,8 @@ pub async fn main() { .await .unwrap(); + return; + // Sanity checks for (_, chunk_data) in manifest.chunks { for file in chunk_data.files { diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 8f7f9e50..02294aa0 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -1,11 +1,15 @@ use std::{ - collections::HashMap, future::Future, path::Path, sync::{ - Arc, atomic::{AtomicU64, Ordering} - } + collections::HashMap, + future::Future, + mem, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; -use anyhow::{anyhow, Error}; -use futures::{stream::FuturesUnordered, StreamExt}; +use anyhow::anyhow; use hex::ToHex as _; use humansize::{format_size, BINARY}; use serde::{Deserialize, Serialize}; @@ -14,6 +18,7 @@ use tokio::{ io::AsyncReadExt as _, join, sync::{Mutex, Semaphore}, + task::JoinSet, }; #[derive(Serialize, Deserialize, Clone)] @@ -39,18 +44,21 @@ pub struct Manifest { pub key: [u8; 16], } -const CHUNK_SIZE: u64 = 1024 * 1024 * 64; +const CHUNK_SIZE: u64 = 1024 * 1024; //* 64; const MAX_FILE_COUNT: usize = 512; -use crate::versions::{create_backend_constructor, types::VersionFile}; +use crate::versions::{ + create_backend_constructor, + types::{VersionBackend, VersionFile}, +}; pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, - reader_semaphore: Option<&Semaphore>, + reader_semaphore: Option>, ) -> anyhow::Result { - let mut backend = + let backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; let required_single_file = backend.require_whole_files(); @@ -137,18 +145,19 @@ pub async fn generate_manifest_rusty( let manifest: Arc>> = Arc::new(Mutex::new(HashMap::new())); let total_manifest_length = Arc::new(AtomicU64::new(0)); - let backend = Arc::new(Mutex::new(backend)); + // SAFETY: we .join_all() the futures using this + let backend: &'static Box = + unsafe { mem::transmute(&backend) }; - let futures: FuturesUnordered>> = - FuturesUnordered::new(); + let mut futures: JoinSet> = JoinSet::new(); let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); let chunks_length = chunks.len(); for (index, chunk) in chunks.into_iter().enumerate() { let send_log = send_log.clone(); - let backend = backend.clone(); let total_manifest_length = total_manifest_length.clone(); let manifest = manifest.clone(); - futures.push(async move { + let reader_semaphore = reader_semaphore.clone(); + futures.spawn(async move { let mut read_buf = vec![0; 1024 * 1024 * 64]; let uuid = uuid::Uuid::new_v4().to_string(); @@ -164,6 +173,8 @@ pub async fn generate_manifest_rusty( let mut chunk_length = 0; + println!("starting chunk {}", index); + for (file, start, length) in chunk { let permit = if let Some(reader_semaphore) = &reader_semaphore { Some(reader_semaphore.acquire().await?) @@ -171,18 +182,20 @@ pub async fn generate_manifest_rusty( None }; - let mut reader = { - let mut backend_lock = backend.lock().await; - let reader = backend_lock.reader(&file, start, start + length).await?; - reader - }; + let mut reader = backend.reader(&file, start, start + length).await?; + + let mut total = 0; loop { let amount = reader.read(&mut read_buf).await?; if amount == 0 { break; } + total += amount; hasher.update(&read_buf[0..amount]); + if total as u64 > length { + panic!("read too much: target {}, got {}", length, total); + } } chunk_length += length; @@ -230,7 +243,7 @@ pub async fn generate_manifest_rusty( progress_sfn((current_progress / total_progress) * 100.0f32) } }, - futures.collect::>>() + futures.join_all() ); let manifest = manifest.lock().await; diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs new file mode 100644 index 00000000..2f8633ad --- /dev/null +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -0,0 +1,115 @@ +use std::{path::PathBuf, task::Poll}; + +use anyhow::{anyhow}; +use async_trait::async_trait; +use libarchive::{ + archive::{Entry, FileType, ReadCompression, ReadFormat}, + reader::{Builder, FileReader, Reader}, +}; +use tokio::io::AsyncRead; + +use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; + +pub struct ZipVersionBackend { + path: PathBuf, +} +impl ZipVersionBackend { + pub fn new(path: PathBuf) -> anyhow::Result { + Ok(Self { path }) + } + + fn open_archive(&self) -> Result { + let mut archive = Builder::new(); + archive.support_format(ReadFormat::All)?; + archive.support_compression(ReadCompression::All)?; + let archive = archive.open_file(&self.path)?; + + Ok(archive) + } +} + +struct ArchiveReader { + archive: FileReader, +} + +impl AsyncRead for ArchiveReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let block = match self.archive.read_block() { + Ok(v) => v, + Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))), + }; + + let block = match block { + Some(v) => v, + None => return Poll::Ready(Ok(())), + }; + + buf.put_slice(block); + return Poll::Ready(Ok(())); + } +} + +#[async_trait] +impl VersionBackend for ZipVersionBackend { + async fn list_files(&self) -> anyhow::Result> { + let mut archive = self.open_archive()?; + let mut results = Vec::new(); + + while let Some(header) = archive.next_header() { + match header.filetype() { + FileType::RegularFile => (), + _ => { + continue; + } + } + results.push(VersionFile { + relative_filename: header.pathname().to_string(), + permission: 0o744, + size: header.size().try_into()?, + }); + println!("storing file {}", header.pathname()); + } + + Ok(results) + } + + async fn reader( + &self, + file: &VersionFile, + _start: u64, + _end: u64, + ) -> anyhow::Result> { + let mut archive = self.open_archive()?; + + // Find entry in archive + loop { + let entry = match archive.next_header() { + Some(v) => v, + None => return Err(anyhow!("entry not found:{}", file.relative_filename)), + }; + if entry.pathname() == file.relative_filename { + break; + } + } + + Ok(Box::new(ArchiveReader { archive })) + } + + async fn peek_file(&self, sub_path: String) -> anyhow::Result { + let files = self.list_files().await?; + let file = files + .iter() + .find(|v| v.relative_filename == sub_path) + .expect("file not found"); + + Ok(file.clone()) + } + + fn require_whole_files(&self) -> bool { + true + } +} diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs deleted file mode 100644 index ecd6876b..00000000 --- a/libraries/droplet/src/versions/backends.rs +++ /dev/null @@ -1,230 +0,0 @@ -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; -use std::{ - fs::{metadata, read_dir}, - io::SeekFrom, - path::{Path, PathBuf}, - process::Stdio, - sync::LazyLock, -}; - -use anyhow::{anyhow, Result}; -use async_trait::async_trait; -use tokio::{ - fs::File, - io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, - process::Command, -}; - -use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; - -pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { - if metadata(path)?.is_dir() { - let paths = read_dir(path)?; - for path_result in paths { - let full_path = path_result?.path(); - if metadata(&full_path)?.is_dir() { - _list_files(vec, &full_path)?; - } else { - vec.push(full_path); - } - } - }; - - Ok(()) -} - -#[derive(Clone)] -pub struct PathVersionBackend { - pub base_dir: PathBuf, -} - -#[async_trait] -impl VersionBackend for PathVersionBackend { - async fn list_files(&mut self) -> anyhow::Result> { - let mut vec = Vec::new(); - _list_files(&mut vec, &self.base_dir)?; - - let mut results = Vec::new(); - - for pathbuf in vec.iter() { - let relative = pathbuf.strip_prefix(self.base_dir.clone())?; - - results.push( - self.peek_file( - relative - .to_str() - .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? - .to_owned(), - ) - .await?, - ); - } - - Ok(results) - } - - async fn reader( - &mut self, - file: &VersionFile, - start: u64, - end: u64, - ) -> anyhow::Result> { - let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; - - if start != 0 { - file.seek(SeekFrom::Start(start)).await?; - } - - if end != 0 { - return Ok(Box::new(file.take(end - start))); - } - - Ok(Box::new(file)) - } - - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let pathbuf = self.base_dir.join(sub_path.clone()); - if !pathbuf.exists() { - return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy())); - }; - - let file = File::open(pathbuf.clone()).await?; - let metadata = file.try_clone().await?.metadata().await?; - let permission_object = metadata.permissions(); - let permissions = { - let perm: u32; - #[cfg(target_family = "unix")] - { - perm = permission_object.mode(); - } - #[cfg(not(target_family = "unix"))] - { - perm = 0 - } - perm - }; - - Ok(VersionFile { - relative_filename: sub_path, - permission: permissions, - size: metadata.len(), - }) - } - - fn require_whole_files(&self) -> bool { - false - } -} - -pub static SEVEN_ZIP_INSTALLED: LazyLock = - LazyLock::new(|| std::process::Command::new("7z").output().is_ok()); -// https://7-zip.opensource.jp/chm/general/formats.htm -// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above -pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ - "7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz", - "txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a", - "deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt", - "cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx", - "hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz", - "mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c", - "squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z", - "taz", -]; - -#[derive(Clone)] -pub struct ZipVersionBackend { - path: String, -} -impl ZipVersionBackend { - pub fn new(path: PathBuf) -> anyhow::Result { - Ok(Self { - path: path.to_str().expect("invalid utf path").to_owned(), - }) - } -} - -#[async_trait] -impl VersionBackend for ZipVersionBackend { - async fn list_files(&mut self) -> anyhow::Result> { - let mut list_command = Command::new("7z"); - list_command.args(vec!["l", &self.path]); - let result = list_command.output().await?; - if !result.status.success() { - return Err(anyhow!( - "failed to list files: code {:?}", - result.status.code() - )); - } - let raw_result = String::from_utf8(result.stdout)?; - let mut lines = raw_result.split("\n").skip(11); - - let mut column_lines = lines - .find(|v| v.starts_with('-')) - .ok_or(anyhow!("invalid 7z output"))? - .chars(); - let file_lines = lines.take_while(|v| !v.starts_with("-")); - - /* - Date Time Attr Size Compressed Name - ------------------- ----- ------------ ------------ ------------------------ - */ - let datetime_pos = 0usize; - let attrs_pos = datetime_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let size_pos = attrs_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let compressed_size_pos = size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let name_pos = compressed_size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - - let mut results = Vec::new(); - for file in file_lines { - let name = file[name_pos..].trim(); - let size = file[size_pos..compressed_size_pos].trim(); - - let size = str::parse::(size)?; - - let version_file = VersionFile { - relative_filename: name.to_string(), - permission: 0o744, - size, - }; - - results.push(version_file); - } - - Ok(results) - } - - async fn reader( - &mut self, - file: &VersionFile, - _start: u64, - _end: u64, - ) -> anyhow::Result> { - let mut read_command = Command::new("7z"); - read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); - let mut output = read_command - .stdout(Stdio::piped()) - .spawn() - .expect("failed to spawn 7z"); - let stdout = output - .stdout - .take() - .ok_or(anyhow!("no stdout on 7zip process"))?; - let reader = BufReader::new(stdout); - Ok(Box::new(reader)) - } - - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let files = self.list_files().await?; - let file = files - .iter() - .find(|v| v.relative_filename == sub_path) - .expect("file not found"); - - Ok(file.clone()) - } - - fn require_whole_files(&self) -> bool { - true - } -} diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 4127d9d2..5549ff7c 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -1,15 +1,37 @@ -use std::path::Path; +use std::{ + fs::{metadata, read_dir}, + path::{Path, PathBuf}, +}; use anyhow::Result; use crate::versions::{ - backends::{ - PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED, SUPPORTED_FILE_EXTENSIONS, - }, - types::VersionBackend, + archive_backend::ZipVersionBackend, path_backend::PathVersionBackend, types::VersionBackend, }; -pub mod backends; +pub mod archive_backend; +pub mod path_backend; + +pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { + if metadata(path)?.is_dir() { + let paths = read_dir(path)?; + for path_result in paths { + let full_path = path_result?.path(); + if metadata(&full_path)?.is_dir() { + _list_files(vec, &full_path)?; + } else { + vec.push(full_path); + } + } + }; + + Ok(()) +} + +const SUPPORTED_FILE_EXTENSIONS: [&'static str; 11] = [ + "tar", "pax", "cpio", "zip", "jar", "ar", "xar", "rar", "rpm", "7z", "iso", +]; + pub mod types; pub fn create_backend_constructor<'a>( path: &Path, @@ -26,27 +48,17 @@ pub fn create_backend_constructor<'a>( })); }; - if *SEVEN_ZIP_INSTALLED { - /* - Slow 7zip integrity test - let mut test = Command::new("7z"); - test.args(vec!["t", path.to_str().expect("invalid utf path")]); - let status = test.status().ok()?; - if status.code().unwrap_or(1) == 0 { - let buf = path.to_path_buf(); - return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); - } - */ - // Fast filename-based test - if let Some(extension) = path.extension().and_then(|v| v.to_str()) { - let supported = SUPPORTED_FILE_EXTENSIONS - .iter() - .any(|v| **v == *extension); - if supported { - let buf = path.to_path_buf(); - return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); - } - } + let file_extension = path + .extension() + .map(|v| v.to_str()) + .flatten()?; + + if SUPPORTED_FILE_EXTENSIONS + .iter() + .any(|v| *v == file_extension) + { + let buf = path.to_path_buf(); + return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); } None diff --git a/libraries/droplet/src/versions/path_backend.rs b/libraries/droplet/src/versions/path_backend.rs new file mode 100644 index 00000000..5a0247fa --- /dev/null +++ b/libraries/droplet/src/versions/path_backend.rs @@ -0,0 +1,98 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{ + io::SeekFrom, + path::PathBuf, +}; + +use anyhow::anyhow; +use async_trait::async_trait; +use tokio::{ + fs::File, + io::{AsyncReadExt as _, AsyncSeekExt as _}, +}; + +#[derive(Clone)] +pub struct PathVersionBackend { + pub base_dir: PathBuf, +} + +use crate::versions::{_list_files, types::{MinimumFileObject, VersionBackend, VersionFile}}; + +#[async_trait] +impl VersionBackend for PathVersionBackend { + async fn list_files(&self) -> anyhow::Result> { + let mut vec = Vec::new(); + _list_files(&mut vec, &self.base_dir)?; + + let mut results = Vec::new(); + + for pathbuf in vec.iter() { + let relative = pathbuf.strip_prefix(self.base_dir.clone())?; + + results.push( + self.peek_file( + relative + .to_str() + .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? + .to_owned(), + ) + .await?, + ); + } + + Ok(results) + } + + async fn reader( + &self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; + + if start != 0 { + file.seek(SeekFrom::Start(start)).await?; + } + + if end != 0 { + return Ok(Box::new(file.take(end - start))); + } + + Ok(Box::new(file)) + } + + async fn peek_file(&self, sub_path: String) -> anyhow::Result { + let pathbuf = self.base_dir.join(sub_path.clone()); + if !pathbuf.exists() { + return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy())); + }; + + let file = File::open(pathbuf.clone()).await?; + let metadata = file.try_clone().await?.metadata().await?; + let permission_object = metadata.permissions(); + let permissions = { + let perm: u32; + #[cfg(target_family = "unix")] + { + perm = permission_object.mode(); + } + #[cfg(not(target_family = "unix"))] + { + perm = 0 + } + perm + }; + + Ok(VersionFile { + relative_filename: sub_path, + permission: permissions, + size: metadata.len(), + }) + } + + fn require_whole_files(&self) -> bool { + false + } +} diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 5a7cfd95..317408da 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -1,7 +1,6 @@ use std::fmt::Debug; use async_trait::async_trait; -use dyn_clone::DynClone; use tokio::io::AsyncRead; #[derive(Debug, Clone)] @@ -16,16 +15,14 @@ impl MinimumFileObject for T {} #[async_trait] -pub trait VersionBackend: DynClone { +pub trait VersionBackend { fn require_whole_files(&self) -> bool; - async fn list_files(&mut self) -> anyhow::Result>; - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + async fn list_files(&self) -> anyhow::Result>; + async fn peek_file(&self, sub_path: String) -> anyhow::Result; async fn reader( - &mut self, + &self, file: &VersionFile, start: u64, end: u64, ) -> anyhow::Result>; } - -dyn_clone::clone_trait_object!(VersionBackend); diff --git a/libraries/droplet/src/vm/mod.rs b/libraries/droplet/src/vm/mod.rs new file mode 100644 index 00000000..e69de29b From b2b3ee7d3c8ed3f69c5b8460904cf95a5e291e91 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:19:05 +1100 Subject: [PATCH 37/50] version bump --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 157e5988..1de698f3 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index eaaddaf9..eb6ec359 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.15.1" +version = "0.16.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" From c4bfe3357d5cd98f9d8bdd4aa7452701fb3268a0 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:21:23 +1100 Subject: [PATCH 38/50] fix: use renamed package --- libraries/droplet/Cargo.lock | 4 ++-- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/libarchive-rust | 2 +- libraries/droplet/src/lib.rs | 2 +- libraries/droplet/src/versions/archive_backend.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 1de698f3..91ac71fa 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -258,7 +258,7 @@ dependencies = [ "getrandom 0.3.4", "hex", "humansize", - "libarchive", + "libarchive-drop", "rcgen", "ring", "serde", @@ -465,7 +465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "libarchive" +name = "libarchive-drop" version = "0.1.1" dependencies = [ "libarchive3-sys", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index eb6ec359..2e21ce81 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -29,7 +29,7 @@ uuid = { version = "1.19.0", features = ["v4"] } sha2 = "0.10.9" futures = "0.3.31" getrandom = "0.3.4" -libarchive = { version = "*", path = "./libarchive-rust" } +libarchive-drop = { version = "*", path = "./libarchive-rust" } speedometer = "0.2.2" [dependencies.x509-parser] diff --git a/libraries/droplet/libarchive-rust b/libraries/droplet/libarchive-rust index 279fd5f7..c694faba 160000 --- a/libraries/droplet/libarchive-rust +++ b/libraries/droplet/libarchive-rust @@ -1 +1 @@ -Subproject commit 279fd5f727d5af50771cdfd660d657dff52c42f7 +Subproject commit c694faba63d2fc3d79c4fde70ebd699adcac9aae diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 8ad90277..3459ecf0 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -7,7 +7,7 @@ pub mod versions; pub mod manifest; pub mod vm; -extern crate libarchive; +extern crate libarchive_drop; #[cfg(test)] pub mod tests; diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs index 2f8633ad..5729d1cc 100644 --- a/libraries/droplet/src/versions/archive_backend.rs +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -2,7 +2,7 @@ use std::{path::PathBuf, task::Poll}; use anyhow::{anyhow}; use async_trait::async_trait; -use libarchive::{ +use libarchive_drop::{ archive::{Entry, FileType, ReadCompression, ReadFormat}, reader::{Builder, FileReader, Reader}, }; From e7d69fc98d4c3d9fe3b5a1008cf1a6b9d8509b32 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:23:26 +1100 Subject: [PATCH 39/50] fix: update submodule type --- libraries/droplet/.gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/droplet/.gitmodules b/libraries/droplet/.gitmodules index 8dfab4fd..d471795d 100644 --- a/libraries/droplet/.gitmodules +++ b/libraries/droplet/.gitmodules @@ -1,3 +1,3 @@ [submodule "libarchive-rust"] path = libarchive-rust - url = git@github.com:Drop-OSS/libarchive-rust.git + url = https://github.com/Drop-OSS/libarchive-rust.git From 642e3363a717217e2062d8c3ea1b2655f779f0c6 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:28:27 +1100 Subject: [PATCH 40/50] fix: remove debug info --- libraries/droplet/src/manifest.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 02294aa0..9fe97ffc 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -44,7 +44,7 @@ pub struct Manifest { pub key: [u8; 16], } -const CHUNK_SIZE: u64 = 1024 * 1024; //* 64; +const CHUNK_SIZE: u64 = 1024 * 1024 * 64; const MAX_FILE_COUNT: usize = 512; use crate::versions::{ @@ -173,8 +173,6 @@ pub async fn generate_manifest_rusty( let mut chunk_length = 0; - println!("starting chunk {}", index); - for (file, start, length) in chunk { let permit = if let Some(reader_semaphore) = &reader_semaphore { Some(reader_semaphore.acquire().await?) From b513699ad9279556a0a6e8b65cded5f42e9d1383 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:28:43 +1100 Subject: [PATCH 41/50] bump version --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 91ac71fa..da06adc3 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 2e21ce81..ad409cac 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.16.0" +version = "0.16.1" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" From f5c4cbe18f6d7981d8bc7439b55584327d434993 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:31:11 +1100 Subject: [PATCH 42/50] fix: remove final debug statement --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/versions/archive_backend.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index da06adc3..02718197 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.16.1" +version = "0.16.2" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index ad409cac..1497a77d 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.16.1" +version = "0.16.2" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs index 5729d1cc..36890a86 100644 --- a/libraries/droplet/src/versions/archive_backend.rs +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -71,7 +71,6 @@ impl VersionBackend for ZipVersionBackend { permission: 0o744, size: header.size().try_into()?, }); - println!("storing file {}", header.pathname()); } Ok(results) From 4cdcc3834954797762af83bcc34b922564ece7d5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:09:22 +1100 Subject: [PATCH 43/50] feat: add tests --- libraries/droplet/.github/workflows/test.yml | 42 ++++++ libraries/droplet/Cargo.lock | 124 +++++++++++++----- libraries/droplet/Cargo.toml | 1 + libraries/droplet/src/lib.rs | 2 +- libraries/droplet/src/ssl.rs | 6 +- libraries/droplet/src/tests.rs | 50 +++++-- .../droplet/src/versions/archive_backend.rs | 2 +- libraries/droplet/src/versions/mod.rs | 5 +- .../droplet/src/versions/path_backend.rs | 15 ++- libraries/droplet/src/versions/types.rs | 1 - libraries/droplet/src/vm/mod.rs | 1 + .../droplet/testfiles/bare_files/dotfile.7z | Bin 0 -> 339 bytes .../testfiles/bare_files/emoji_filename.7z | Bin 0 -> 347 bytes .../testfiles/bare_files/empty_file.7z | Bin 0 -> 98 bytes .../file and archive with spaces.7z | Bin 0 -> 371 bytes .../file_with_space_in_extension.7z | Bin 0 -> 363 bytes .../testfiles/bare_files/file_with_spaces.7z | Bin 0 -> 371 bytes .../bare_files/file_without_extension.7z | Bin 0 -> 379 bytes ...filename_only_environment_variable_name.7z | Bin 0 -> 82 bytes .../bare_files/filename_with_asterisk.7z | Bin 0 -> 98 bytes .../bare_files/filename_with_backslashes.7z | Bin 0 -> 379 bytes .../bare_files/filename_with_colon.7z | Bin 0 -> 98 bytes .../bare_files/filename_with_dollar.7z | Bin 0 -> 98 bytes ...filename_with_environment_variable_name.7z | Bin 0 -> 106 bytes .../filename_with_full_windows_path_name.7z | Bin 0 -> 122 bytes .../bare_files/filename_with_lessthan.7z | Bin 0 -> 98 bytes .../bare_files/filename_with_newline.7z | Bin 0 -> 98 bytes .../filename_with_spaces_and_backslashes.7z | Bin 0 -> 419 bytes .../testfiles/bare_files/filename_with_tab.7z | Bin 0 -> 98 bytes ..._with_windows_environment_variable_name.7z | Bin 0 -> 90 bytes .../testfiles/bare_files/standard_archive.7z | Bin 0 -> 347 bytes .../testfiles/bare_files/unicode_filename.7z | Bin 0 -> 347 bytes .../bare_files/windows_reserved_file_name.7z | Bin 0 -> 82 bytes .../subdirectories/dotsubdirectory.7z | Bin 0 -> 413 bytes .../subdirectories/empty_dotsubdirectory.7z | Bin 0 -> 106 bytes .../subdirectories/empty_subdirectory.7z | Bin 0 -> 98 bytes ...ectory_and_file_with_double_backslashes.7z | Bin 0 -> 449 bytes .../subdirectory_and_file_with_spaces.7z | Bin 0 -> 435 bytes ...bdirectory_file_with_double_backslashes.7z | Bin 0 -> 414 bytes .../subdirectory_with_backslashes.7z | Bin 0 -> 130 bytes .../subdirectory_with_double_backslashes.7z | Bin 0 -> 138 bytes .../subdirectory_with_single_backslashes.7z | Bin 0 -> 438 bytes .../subdirectory_with_symlink_to_self.7z | Bin 0 -> 263 bytes 43 files changed, 190 insertions(+), 59 deletions(-) create mode 100644 libraries/droplet/.github/workflows/test.yml create mode 100644 libraries/droplet/testfiles/bare_files/dotfile.7z create mode 100644 libraries/droplet/testfiles/bare_files/emoji_filename.7z create mode 100644 libraries/droplet/testfiles/bare_files/empty_file.7z create mode 100644 libraries/droplet/testfiles/bare_files/file and archive with spaces.7z create mode 100644 libraries/droplet/testfiles/bare_files/file_with_space_in_extension.7z create mode 100644 libraries/droplet/testfiles/bare_files/file_with_spaces.7z create mode 100644 libraries/droplet/testfiles/bare_files/file_without_extension.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_only_environment_variable_name.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_asterisk.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_backslashes.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_colon.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_dollar.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_environment_variable_name.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_full_windows_path_name.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_lessthan.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_newline.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_spaces_and_backslashes.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_tab.7z create mode 100644 libraries/droplet/testfiles/bare_files/filename_with_windows_environment_variable_name.7z create mode 100644 libraries/droplet/testfiles/bare_files/standard_archive.7z create mode 100644 libraries/droplet/testfiles/bare_files/unicode_filename.7z create mode 100644 libraries/droplet/testfiles/bare_files/windows_reserved_file_name.7z create mode 100644 libraries/droplet/testfiles/subdirectories/dotsubdirectory.7z create mode 100644 libraries/droplet/testfiles/subdirectories/empty_dotsubdirectory.7z create mode 100644 libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_double_backslashes.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_spaces.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_file_with_double_backslashes.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_backslashes.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_single_backslashes.7z create mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_symlink_to_self.7z diff --git a/libraries/droplet/.github/workflows/test.yml b/libraries/droplet/.github/workflows/test.yml new file mode 100644 index 00000000..049d389f --- /dev/null +++ b/libraries/droplet/.github/workflows/test.yml @@ -0,0 +1,42 @@ +name: Rust CI + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + ci: + name: Build, Test, Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Run Clippy (lint) + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Run tests + run: cargo test --all-features --all --verbose \ No newline at end of file diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 02718197..cf7b8dac 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -61,8 +61,8 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", "synstructure 0.13.1", ] @@ -73,8 +73,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", "synstructure 0.13.1", ] @@ -85,8 +85,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -96,8 +96,8 @@ version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -242,8 +242,8 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -265,6 +265,7 @@ dependencies = [ "serde_json", "sha2", "speedometer", + "test-generator", "time", "tokio", "uuid", @@ -293,8 +294,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", "synstructure 0.12.6", ] @@ -353,8 +354,8 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -427,6 +428,12 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "hex" version = "0.4.3" @@ -637,6 +644,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.94" @@ -646,13 +662,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.94", ] [[package]] @@ -747,8 +772,8 @@ version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -806,14 +831,25 @@ dependencies = [ "failure", ] +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] @@ -823,8 +859,8 @@ version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "unicode-ident", ] @@ -834,10 +870,10 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 1.0.109", - "unicode-xid", + "unicode-xid 0.2.6", ] [[package]] @@ -846,11 +882,23 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] +[[package]] +name = "test-generator" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b23be2add79223226e1cb6446cb3e37506a5927089870687a0f1149bb7a073a" +dependencies = [ + "glob", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -875,8 +923,8 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -886,8 +934,8 @@ version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -943,8 +991,8 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", ] @@ -960,6 +1008,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1023,7 +1077,7 @@ version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ - "quote", + "quote 1.0.40", "wasm-bindgen-macro-support", ] @@ -1034,8 +1088,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", - "proc-macro2", - "quote", + "proc-macro2 1.0.94", + "quote 1.0.40", "syn 2.0.117", "wasm-bindgen-shared", ] diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 1497a77d..0417f24b 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -31,6 +31,7 @@ futures = "0.3.31" getrandom = "0.3.4" libarchive-drop = { version = "*", path = "./libarchive-rust" } speedometer = "0.2.2" +test-generator = "0.3.1" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 3459ecf0..666a17bc 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -2,9 +2,9 @@ #![feature(impl_trait_in_bindings)] pub mod file_utils; +pub mod manifest; pub mod ssl; pub mod versions; -pub mod manifest; pub mod vm; extern crate libarchive_drop; diff --git a/libraries/droplet/src/ssl.rs b/libraries/droplet/src/ssl.rs index 6f59e984..3a596497 100644 --- a/libraries/droplet/src/ssl.rs +++ b/libraries/droplet/src/ssl.rs @@ -1,5 +1,6 @@ use rcgen::{ - CertificateParams, DistinguishedName, Error, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData, SubjectPublicKeyInfo + CertificateParams, DistinguishedName, Error, IsCa, KeyPair, KeyUsagePurpose, PublicKeyData, + SubjectPublicKeyInfo, }; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, VerificationAlgorithm}; @@ -42,8 +43,7 @@ pub fn generate_client_certificate( root_ca: String, root_ca_private: String, ) -> Result, rcgen::Error> { - let root_key_pair = - KeyPair::from_pem(&root_ca_private)?; + let root_key_pair = KeyPair::from_pem(&root_ca_private)?; let certificate_params = CertificateParams::from_ca_cert_pem(&root_ca)?; let root_ca = CertificateParams::self_signed(certificate_params, &root_key_pair)?; diff --git a/libraries/droplet/src/tests.rs b/libraries/droplet/src/tests.rs index 99edf35b..d8d097f4 100644 --- a/libraries/droplet/src/tests.rs +++ b/libraries/droplet/src/tests.rs @@ -1,11 +1,45 @@ -use std::path::PathBuf; +#![cfg(test)] +extern crate test_generator; -use crate::versions::create_backend_constructor; +use std::path::Path; -#[tokio::test] -pub async fn test_7z_list() { - let zip_path = "/home/decduck/Dev/droplet/assets/TheGame.zip"; - let mut backend = create_backend_constructor(&PathBuf::from(zip_path)).unwrap()().unwrap(); - let files = backend.list_files().await.unwrap(); - tokio::fs::write("./test.txt", format!("{:?}", files)).await.unwrap(); +use serde_json::json; +use test_generator::test_resources; +use tokio::{fs::File, io::AsyncWriteExt}; + +use crate::manifest::generate_manifest_rusty; + +#[test_resources("testfiles/**/*.7z")] +fn manifest_gen(resource: &str) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to create tokio runtime"); + + runtime.block_on(async move { + let filepath = Path::new(resource); + let manifest = generate_manifest_rusty( + &filepath, + |_| {}, + |message| { + println!("({}) {}", filepath.display(), message); + }, + None, + ) + .await + .expect(&format!( + "failed to generate manifest for {}", + filepath.display() + )); + + let mut output_path = filepath.to_path_buf(); + output_path.set_extension("json"); + + let mut file = File::create(output_path) + .await + .expect("failed to open output path"); + file.write_all(json!(manifest).to_string().as_bytes()) + .await + .expect("failed to write output"); + }); } diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs index 36890a86..bf1604fa 100644 --- a/libraries/droplet/src/versions/archive_backend.rs +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, task::Poll}; -use anyhow::{anyhow}; +use anyhow::anyhow; use async_trait::async_trait; use libarchive_drop::{ archive::{Entry, FileType, ReadCompression, ReadFormat}, diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 5549ff7c..e73be877 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -48,10 +48,7 @@ pub fn create_backend_constructor<'a>( })); }; - let file_extension = path - .extension() - .map(|v| v.to_str()) - .flatten()?; + let file_extension = path.extension().map(|v| v.to_str()).flatten()?; if SUPPORTED_FILE_EXTENSIONS .iter() diff --git a/libraries/droplet/src/versions/path_backend.rs b/libraries/droplet/src/versions/path_backend.rs index 5a0247fa..e3b70570 100644 --- a/libraries/droplet/src/versions/path_backend.rs +++ b/libraries/droplet/src/versions/path_backend.rs @@ -1,9 +1,6 @@ #[cfg(unix)] use std::os::unix::fs::PermissionsExt; -use std::{ - io::SeekFrom, - path::PathBuf, -}; +use std::{io::SeekFrom, path::PathBuf}; use anyhow::anyhow; use async_trait::async_trait; @@ -17,7 +14,10 @@ pub struct PathVersionBackend { pub base_dir: PathBuf, } -use crate::versions::{_list_files, types::{MinimumFileObject, VersionBackend, VersionFile}}; +use crate::versions::{ + _list_files, + types::{MinimumFileObject, VersionBackend, VersionFile}, +}; #[async_trait] impl VersionBackend for PathVersionBackend { @@ -34,7 +34,10 @@ impl VersionBackend for PathVersionBackend { self.peek_file( relative .to_str() - .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? + .ok_or(anyhow!( + "Could not parse path: {}", + relative.to_string_lossy() + ))? .to_owned(), ) .await?, diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 317408da..d80c1b36 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -13,7 +13,6 @@ pub struct VersionFile { pub trait MinimumFileObject: AsyncRead + Send + Unpin {} impl MinimumFileObject for T {} - #[async_trait] pub trait VersionBackend { fn require_whole_files(&self) -> bool; diff --git a/libraries/droplet/src/vm/mod.rs b/libraries/droplet/src/vm/mod.rs index e69de29b..8b137891 100644 --- a/libraries/droplet/src/vm/mod.rs +++ b/libraries/droplet/src/vm/mod.rs @@ -0,0 +1 @@ + diff --git a/libraries/droplet/testfiles/bare_files/dotfile.7z b/libraries/droplet/testfiles/bare_files/dotfile.7z new file mode 100644 index 0000000000000000000000000000000000000000..9c278ed2c4022b2b142c33416a7c403283cd7167 GIT binary patch literal 339 zcmXr7+Ou9=hJmGf%lyWN3=j|mrEB7%Jl_BR&u}xAfg%6z|No1(|97|Z5G}Gy4BRnE z5+V!aX$F?c;JXr_s zm3XE+dKnYEiRH$37oOGeyt!^KpB^sj*gh?M%{JeMnX>u6y*mVQoKAPx$1wC?@LR8@ zF{M!6wqNC8!Mx?l@ozLvw_cn&qpMD9;wSae?~(`DX86wddt?383H(P?gMWOTXlBT| zbi=#+g9X0sJQpVlu3DQQyX)LWb%~hBwwGx^5&K@2t%-c{G@6y;=kZ%>|NIWp`u=39 zDgy%}3mXF?XTw7Vc5X%nMny$N0iFv$|8sCL%Dl2z&cMLRD9H&84+aK7ZU#MuRE7$M g5}=SIg9sNRgUIYzA*WW}VH9O!WKd{W(#XI70HF_tdH?_b literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/emoji_filename.7z b/libraries/droplet/testfiles/bare_files/emoji_filename.7z new file mode 100644 index 0000000000000000000000000000000000000000..f65e7d1e5a5471455caa6c0423532b2028b932a1 GIT binary patch literal 347 zcmXr7+Ou9=hJi)qOvaao3=j|nr7aF$W_bVqKf}#f28R5<|Nk%E{@>lsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%otY#Lc@|q1_l655Q#JZ literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/empty_file.7z b/libraries/droplet/testfiles/bare_files/empty_file.7z new file mode 100644 index 0000000000000000000000000000000000000000..0e4fa0bc39524c572980cea39585896e93e9e398 GIT binary patch literal 98 zcmXr7+Ou9=hJmHokoOz|6gWZY#Mgft8CelsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%oKf}#f28R5<|Nk%E{@>lsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%olsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%oW?iCoiV{i9YkjNTjTPIh(q>M!QKS-jI4}&j1ByZ4U$X@41&B2DhwVB{tUhhu0S;+ aT#O8if4v>fuDru2%Eri`(6FSDfdK$1y$&A$ literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_asterisk.7z b/libraries/droplet/testfiles/bare_files/filename_with_asterisk.7z new file mode 100644 index 0000000000000000000000000000000000000000..9435c06c1cee49f64721514af94ae5ba1606bcfa GIT binary patch literal 98 zcmXr7+Ou9=hJocdqwf<2C~$((Ec?%WXJlpMV{G7OY>;GPU=WmLNMp!k$YDrj&|=7A oNMy(bvh;wsf}sScM}&)!VTxYD`LiqUFp9DSvC%d|Q^%wKrtY7xHM_DlVL8Qdd(ymA4KekP&=gB&7 zuf#Lu(aV_NO)NLQyYQ@z=goC{`SfsE$M$LAYqt45%#_Xl?cE`e<8->iK8B(Hg5P>I zjVXolw*4v(3+63Xj(?+Zy7l7J8C`W+6F;e!ewRGJHp6$u-y7?%PT)VH8vNtyL^DIy zr5oPmA1v^7=eamhaMjub*iRH5;GPU=WmLNMp!k$YDrjuwuw# oNMy(bvh;wsf}sScM}&)!;R>T&^4XPl7)99_85A0pG%_#%0KY5~jsO4v literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_dollar.7z b/libraries/droplet/testfiles/bare_files/filename_with_dollar.7z new file mode 100644 index 0000000000000000000000000000000000000000..7d63273ddad953335070fdcf4e7786c0443133fc GIT binary patch literal 98 zcmXr7+Ou9=hJoeER>c?wC~$((rmsyGGqN)BF*fisHb^otFbK*rq%mYNR2V!M v{26>1Tp98h5*c!VB6>hv!B7I!E5gOda7M41`|Qd)jG}Ce37`7BC literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_full_windows_path_name.7z b/libraries/droplet/testfiles/bare_files/filename_with_full_windows_path_name.7z new file mode 100644 index 0000000000000000000000000000000000000000..9a81afaa9945c8493fb4d839301bf66fb304a36c GIT binary patch literal 122 zcmXr7+Ou9=hJocw977xf6huL3O|dUJjI4}&j1ByZ4U$X@41$IX&J0!zF%01hnGAUh zDGd1xY#8zw n5*c!VEIlBuU?>6V5#eHF2&h@N{Orm*jG}Ce3qstTB literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_newline.7z b/libraries/droplet/testfiles/bare_files/filename_with_newline.7z new file mode 100644 index 0000000000000000000000000000000000000000..8bda74eaa0cdb9fc39dc4a4bc25185a26165edac GIT binary patch literal 98 zcmXr7+Ou9=hJocr|3y&-C~$((50~zm%*e{f$JoHn*dWQoz#u5gkj9Y7ki(G5z{QZq okjRh=Wa$BM1w#o?j|dkdLqyo#)n``TVH9O!WKd{W(#XI70Or~hhX4Qo literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_spaces_and_backslashes.7z b/libraries/droplet/testfiles/bare_files/filename_with_spaces_and_backslashes.7z new file mode 100644 index 0000000000000000000000000000000000000000..0d970e530590c7aca7fedafc2d6421b6d715e935 GIT binary patch literal 419 zcmXr7+Ou9=hJj^blY+xT1_)RLr7f=nynFxuKf}#f28R5<|Nk%E{@>lsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%o;GPU=WmLNMp!k$YDrj;AF^S oNMy(bvh;wsf}sScM}&)!Vfv2?=g+LX!zjwe$e_@$q>+IE0Oko5Qvd(} literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/filename_with_windows_environment_variable_name.7z b/libraries/droplet/testfiles/bare_files/filename_with_windows_environment_variable_name.7z new file mode 100644 index 0000000000000000000000000000000000000000..2f2c2612cc5f6ad3fed0538adfd69c4dc08fa75d GIT binary patch literal 90 zcmXr7+Ou9=hJi(G!lkneP+$e6cT@{pWn^XKV{G7OY>;GPU=ZYIP-O^Z2xf3)2m-Pg g7$jLhN=3LB8S2(ZHlAI1hf$P`kwKwhNh1RT0OSr3DF6Tf literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/standard_archive.7z b/libraries/droplet/testfiles/bare_files/standard_archive.7z new file mode 100644 index 0000000000000000000000000000000000000000..8d0c97cf1e61735bbaaf9b148c637abe5f4700d3 GIT binary patch literal 347 zcmXr7+Ou9=hJoea`y=cR86Y4EN;_G8xcL76e}SvC%d|Q^%wKrtY7xHM_DlVL8Qdd(ymA4KekP&=gB&7 zuf#Lu(aV_NO)NLQyYQ@z=goC{`SfsE$M$LAYqt45%#_Xl?cE`e<8->iK8B(Hg5P>I zjVXolw*4v(3+63Xj(?+Zy7l7J8C`W+6F;e!ewRGJHp6$u-y7?%PT)VH8vNtyL^DIy zr5oPmA1v^7=eamhaMjub*C^Rf-WMBXQ_LYeS literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/bare_files/unicode_filename.7z b/libraries/droplet/testfiles/bare_files/unicode_filename.7z new file mode 100644 index 0000000000000000000000000000000000000000..57999040802b169095d4a739ebd27391bddfdc91 GIT binary patch literal 347 zcmXr7+Ou9=hJoeugU=TpGC)8Sl%61bYr^~g{~2z^GBD)-{r`XQ_W$m79->8-iGe#N zX*^rp#{To&6S=0#`bVv77`->vo$TuJ)nCkevwqp*9%aGc2ays-OS>MG|JXLAo+s zG^P~F+xDwGESR@kIsT2t>DG%=XLQwRP5h)@`d#t>+YH|se{Zb6I)VR)YVeP*6U_`+ zmu`5Mf3U#Uo#*02!BuM$WOtq0s4fu`+4eFmC}Q8svNe%Uo<_5B{5*bZ?VsO4THl{c zRb^mcWMN}qplg_TZ!zjwe$e_@$q>+IE0A)T8&j0`b literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/dotsubdirectory.7z b/libraries/droplet/testfiles/subdirectories/dotsubdirectory.7z new file mode 100644 index 0000000000000000000000000000000000000000..44e6e5fc54e3f226ea7bb47c11b8e3d658db9105 GIT binary patch literal 413 zcmV;O0b>3)dc3bE8~_Bcy|UO_0RR910000Y000000001vY#+?;|Nj8lT>t=Y{r~@? zxBo{zNELE9VNtk}DCVPv2l?*g9)i;^#X39zOR;{*ii%AyqYc@ArsIqw5mVq<7{zsp z;&=GAlz$9rz}|F8Wa8;uQ?dlu_e2bE*+Ah_{wkthP?q5H(Uq8rekzgjCw2E3zy_F2nElwV)sPRwB~$qI zkv1^}rLgXA!EjAS4AGGhs;yuhy3VmD7+hI~>1I+`zUg+XS>)wg1qk`a+O7QiQY!c4 zl_dZG004nA2d)p*G0nASB}Z2U(;Cx-G1j(uSVWaH-MzKf8G#FprhuiT8uto))tznT zg6G45JsESjN29KDuPPVbhE3cpJ@WHo4+dmA>FpzH_Tze%nrq*t;dL);UNEfZ+iNdH@Ft0RRCb0|5aAT>uaO01SYp3IW6r Hev$wHyehiG literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/empty_dotsubdirectory.7z b/libraries/droplet/testfiles/subdirectories/empty_dotsubdirectory.7z new file mode 100644 index 0000000000000000000000000000000000000000..40cfafe9410d3a32e3352fc67567c29d4b435131 GIT binary patch literal 106 zcmXr7+Ou9=hJoeiqWMpOJV{DLQ1q%r(Gw3m-Fyu3oFcdSCG9&@n rnG8h?sSL?Lk$fPl5~xg)L4=EuLC-eV^YqF)jG}Ce3<3>r9T^w^Nk9~K literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z b/libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z new file mode 100644 index 0000000000000000000000000000000000000000..c58de4c1a472735a5e8f020fd824eafa3a6aeb87 GIT binary patch literal 98 zcmXr7+Ou9=hJhvWoAgu$C~$((Ps~=-FtRf8F*ZoDf`tU78HyQ78Il-M7%~}(7*ZLM m8A=%PfvifPv?PNF7bAmR(o50PEAKFhvN19UG`w|WU;qGx6cXbA literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_double_backslashes.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_double_backslashes.7z new file mode 100644 index 0000000000000000000000000000000000000000..344b401016f912d50f11ab8cd56e6775aaa047f5 GIT binary patch literal 449 zcmV;y0Y3gWdc3bE8~_CQSHxs~0RR910000Z000000002kV3i~9|Nj8lT>t=Y{r~@? zxBo{zNELE9VNtk}DCVPv2l?*g9)i;^#X39zOR;{*ii%AyqYc@ArsIqw5mVq<7{zsp z;&=GAlz$9rz}|F8Wa8;uQ?dlu_e2bE*+Ah_{wkthP?q5H(Uq8rekzgjCw2E3zy_F2nElwV)sPRwB~$qI zkv1^}rLgXA!EjAS4AGGhs;yuhy3VmD7+hI~>1I+`zUg+XS>)wg1qk`a+O7QiQY!c4 zl_dZG004nA2d)p*G0nASB}Z2U(;Cx-G1j(uSVWaH-MzKf8G#FprhuiH;?yR!_}wtp ze0WwoY+(1A1B6M^noTk&fI2yBwwPM1akE8^WszPsW;f4_6D#*hds8PY%G1(Tbp$&0 z-?RQJV`s!OD+!Q8XzbI(0ZNt?{d2G~M!AYSHrA4DwrYeMHGqSVV}fQs!~zV)vluQI rpveFL02c;;;QuaO01SZ+3IPY^IQ9Sl&t=B& literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_spaces.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_and_file_with_spaces.7z new file mode 100644 index 0000000000000000000000000000000000000000..99db08e731f8fbefdc22eef8f8a5f0b244a23862 GIT binary patch literal 435 zcmV;k0Zjfkdc3bE8~_ApSoH>Q0RR910000Z000000002_>@|_^|Nj8lT>t=Y{r~@? zxBo{zNELE9VNtk}DCVPv2l?*g9)i;^#X39zOR;{*ii%AyqYc@ArsIqw5mVq<7{zsp z;&=GAlz$9rz}|F8Wa8;uQ?dlu_e2bE*+Ah_{wkthP?q5H(Uq8rekzgjCw2E3zy_F2nElwV)sPRwB~$qI zkv1^}rLgXA!EjAS4AGGhs;yuhy3VmD7+hI~>1I+`zUg+XS>)wg1qk`a+O7QiQY!c4 zl_dZG004nA2d)p*G0nASB}Z2U(;Cx-G1j(uSVWaH-MzKf8G#FprhuiH;?yR!kR>#> zW$>ce@m#!d_PYzmJ&mo_s$=|k@}T5Z6mt*L&5!r^Dhh;-=SDwdj3E{JP)(N|Kd!^8 zQWBrt=~!9Q%hBGg=f$N>WWoY>^hKLSY61!!tA-(1Yx$yuO27uuK34o6P d2MYlJ0V4we0R>$E5C8xSfaVGTqD1mP004^*$P54g literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_file_with_double_backslashes.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_file_with_double_backslashes.7z new file mode 100644 index 0000000000000000000000000000000000000000..a04225ecb6fd14003b28b81604cc5af71a9423d1 GIT binary patch literal 414 zcmV;P0b%|(dc3bE8~_CG$xsel0RR910000Y000000002C&%4y`|Nj8lT>t=Y{r~@? zxBo{zNELE9VNtk}DCVPv2l?*g9)i;^#X39zOR;{*ii%AyqYc@ArsIqw5mVq<7{zsp z;&=GAlz$9rz}|F8Wa8;uQ?dlu_e2bE*+Ah_{wkthP?q5H(Uq8rekzgjCw2E3zy_F2nElwV)sPRwB~$qI zkv1^}rLgXA!EjAS4AGGhs;yuhy3VmD7+hI~>1I+`zUg+XS>)wg1qk`a+O7QiQY!c4 zl_dZG004nA2d)p*G0nASB}Z2U(;Cx-G1j(uSVWaH-MzKf8G#FprhuiT8utoWF4ssb zBw-T3AA3?9qFLZuy+LtS6FEnsKu**G=*(fMmq88)l&R7k4J0Tu7 zqUkr$;>eU-aRPFkO8%CnlIW)ijH-AS27uuK33~ts3jqKDBLe{e1zi9T000bto(cgt I2vsQn01dac!T=+rZ@*g7$;isc$Jijr3KkNyWhiDSWk_O3VaQ}CVn}63 zW+-9E2eK*|Vi?MSVkHb2Kvoh%B2Xq9s5S@4Dh7(B0!ao25iUjsJ=Vin1{> L2sFHPWMBXQu^t+1 literal 0 HcmV?d00001 diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z new file mode 100644 index 0000000000000000000000000000000000000000..a0ee82a6b7f69ade57d7e275eeb85a42336e9494 GIT binary patch literal 138 zcmXr7+Ou9=hJnS&u*Zb~3bLT|D&c?27+D$l7#k#6!9s$L48;tk3`qt=Y{r~@? zxBo{zNELE9VNtk}DCVPv2l?*g9)i;^#X39zOR;{*ii%AyqYc@ArsIqw5mVq<7{zsp z;&=GAlz$9rz}|F8Wa8;uQ?dlu_e2bE*+Ah_{wkthP?q5H(Uq8rekzgjCw2E3zy_F2nElwV)sPRwB~$qI zkv1^}rLgXA!EjAS4AGGhs;yuhy3VmD7+hI~>1I+`zUg+XS>)wg1qk`a+O7QiQY!c4 zl_dZG004nA2d)p*G0nASB}Z2U(;Cx-G1j(uSVWaH-MzKf8G#FprhuiH;?yR!hAa@a zW$>ce@m#!d_PYzmJ&mo_s$=|k^UHx$+niSWiG}EVqGF=JWfoY;XIn@FZp=Gb6E_u6 zJlF&@@4eZtNZKDkCM1xZbWwFq^qUtjJ^QPBYnn8y#*sOZa| z_a@cf;^1gdI{anN!H{K6iSwQ)7@yCZbLD5>BBn^~t5+BwPZPe(bLl|OmA#Xib0ad^ zk~d4N%V)4H4RA0FH2ePci=a@h-upF9Or;#s3HMcH4j$a$SF=cLvh38~GVkN1FSQ-$ zd(ZcHPuk6V+jj1x+x54f)_?Klbd-F5uG^kPDf7DepA6fnG1}lSU;nk?M z3$|6g`)nBJxvb)eHg|c^g4X%Zg%TJT#Mu}aIU9~LuyZpqFe)=MGP1@pI505qY@Wo$ LXm{YlaRvqeEl+G^ literal 0 HcmV?d00001 From 275ed25a637a15c5a9a604e088ad50624c48d10c Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:11:09 +1100 Subject: [PATCH 44/50] fix: submodules --- libraries/droplet/.github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/droplet/.github/workflows/test.yml b/libraries/droplet/.github/workflows/test.yml index 049d389f..db2764e6 100644 --- a/libraries/droplet/.github/workflows/test.yml +++ b/libraries/droplet/.github/workflows/test.yml @@ -17,6 +17,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 3 # fix for when this gets triggered by tag + fetch-tags: true + ref: ${{ github.ref }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable From c006549e954a4b9100a5470ae859d19bc1787a92 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:12:02 +1100 Subject: [PATCH 45/50] fix: libarchive fmt and use nightly toolchain --- libraries/droplet/.github/workflows/test.yml | 2 +- libraries/droplet/libarchive-rust | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/droplet/.github/workflows/test.yml b/libraries/droplet/.github/workflows/test.yml index db2764e6..de76641d 100644 --- a/libraries/droplet/.github/workflows/test.yml +++ b/libraries/droplet/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@nightly with: components: rustfmt, clippy diff --git a/libraries/droplet/libarchive-rust b/libraries/droplet/libarchive-rust index c694faba..bf474266 160000 --- a/libraries/droplet/libarchive-rust +++ b/libraries/droplet/libarchive-rust @@ -1 +1 @@ -Subproject commit c694faba63d2fc3d79c4fde70ebd699adcac9aae +Subproject commit bf47426655a9a6c537be753b8c4189191d2ef656 From 94d3dd503a36ba7eeede44a3e045f8536aab4d1a Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:14:25 +1100 Subject: [PATCH 46/50] fix: install libarchive in ci --- libraries/droplet/.github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/droplet/.github/workflows/test.yml b/libraries/droplet/.github/workflows/test.yml index de76641d..02c5c64c 100644 --- a/libraries/droplet/.github/workflows/test.yml +++ b/libraries/droplet/.github/workflows/test.yml @@ -38,6 +38,11 @@ jobs: target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Install libarchive + run: | + sudo apt-get install libarchive-dev -y + + - name: Check formatting run: cargo fmt --all -- --check From 14346301a364f7c28960f7ceb0d9aa818b437bb6 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:24:39 +1100 Subject: [PATCH 47/50] fix: clippy lints --- libraries/droplet/libarchive-rust | 2 +- libraries/droplet/src/main.rs | 37 +------------------ libraries/droplet/src/manifest.rs | 5 +-- libraries/droplet/src/tests.rs | 13 ++++--- .../droplet/src/versions/archive_backend.rs | 2 +- libraries/droplet/src/versions/mod.rs | 8 ++-- 6 files changed, 18 insertions(+), 49 deletions(-) diff --git a/libraries/droplet/libarchive-rust b/libraries/droplet/libarchive-rust index bf474266..fdb73ef2 160000 --- a/libraries/droplet/libarchive-rust +++ b/libraries/droplet/libarchive-rust @@ -1 +1 @@ -Subproject commit bf47426655a9a6c537be753b8c4189191d2ef656 +Subproject commit fdb73ef2dedb031e396153113ff9eee4ee082dc2 diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index c512fe89..b37cc682 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -1,4 +1,4 @@ -use std::{env, os::unix::fs::MetadataExt, path::PathBuf}; +use std::{env, path::PathBuf}; use droplet_rs::manifest::generate_manifest_rusty; use tokio::runtime::Handle; @@ -11,7 +11,7 @@ pub async fn main() { let metrics = Handle::current().metrics(); println!("using {} workers", metrics.num_workers()); - let manifest = generate_manifest_rusty( + let _manifest = generate_manifest_rusty( &target_dir, |progress| println!("PROGRESS: {}", progress), |message| { @@ -21,37 +21,4 @@ pub async fn main() { ) .await .unwrap(); - - return; - - // Sanity checks - for (_, chunk_data) in manifest.chunks { - for file in chunk_data.files { - let path = target_dir.join(file.filename); - if !path.exists() { - panic!("{} doesn't exist", path.display()); - } - - let metadata = path.metadata().expect("failed to fetch metadata"); - let file_size = metadata.size(); - if file.start > file_size as usize { - panic!( - "start for {} doesn't make sense: start: {}, size: {}", - path.display(), - file.start, - file_size - ); - } - - let end_position = file.start + file.length; - if end_position > file_size as usize { - panic!( - "end for {} doesn't make sense: end: {}, size: {}", - path.display(), - end_position, - file_size - ); - } - } - } } diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 9fe97ffc..964f2ec5 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -1,6 +1,5 @@ use std::{ collections::HashMap, - future::Future, mem, path::Path, sync::{ @@ -146,8 +145,8 @@ pub async fn generate_manifest_rusty( let total_manifest_length = Arc::new(AtomicU64::new(0)); // SAFETY: we .join_all() the futures using this - let backend: &'static Box = - unsafe { mem::transmute(&backend) }; + let backend: &'static (dyn VersionBackend + Send + Sync) = + unsafe { mem::transmute(&*backend) }; let mut futures: JoinSet> = JoinSet::new(); let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); diff --git a/libraries/droplet/src/tests.rs b/libraries/droplet/src/tests.rs index d8d097f4..2b69c70c 100644 --- a/libraries/droplet/src/tests.rs +++ b/libraries/droplet/src/tests.rs @@ -19,7 +19,7 @@ fn manifest_gen(resource: &str) { runtime.block_on(async move { let filepath = Path::new(resource); let manifest = generate_manifest_rusty( - &filepath, + filepath, |_| {}, |message| { println!("({}) {}", filepath.display(), message); @@ -27,10 +27,13 @@ fn manifest_gen(resource: &str) { None, ) .await - .expect(&format!( - "failed to generate manifest for {}", - filepath.display() - )); + .unwrap_or_else(|err| { + panic!( + "failed to generate manifest for {}: {:?}", + filepath.display(), + err + ) + }); let mut output_path = filepath.to_path_buf(); output_path.set_extension("json"); diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs index bf1604fa..75f84de6 100644 --- a/libraries/droplet/src/versions/archive_backend.rs +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -49,7 +49,7 @@ impl AsyncRead for ArchiveReader { }; buf.put_slice(block); - return Poll::Ready(Ok(())); + Poll::Ready(Ok(())) } } diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index e73be877..6f3023c8 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -28,11 +28,12 @@ pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { Ok(()) } -const SUPPORTED_FILE_EXTENSIONS: [&'static str; 11] = [ +const SUPPORTED_FILE_EXTENSIONS: [&str; 11] = [ "tar", "pax", "cpio", "zip", "jar", "ar", "xar", "rar", "rpm", "7z", "iso", ]; pub mod types; +#[allow(clippy::type_complexity)] pub fn create_backend_constructor<'a>( path: &Path, ) -> Option Result>>> { @@ -48,11 +49,10 @@ pub fn create_backend_constructor<'a>( })); }; - let file_extension = path.extension().map(|v| v.to_str()).flatten()?; + let file_extension = path.extension().and_then(|v| v.to_str())?; if SUPPORTED_FILE_EXTENSIONS - .iter() - .any(|v| *v == file_extension) + .contains(&file_extension) { let buf = path.to_path_buf(); return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); From 10b66a9e323f99e938cdbd0b81fa3076ebc58167 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:26:11 +1100 Subject: [PATCH 48/50] fix: cargo fmt --- libraries/droplet/src/manifest.rs | 3 +-- libraries/droplet/src/versions/mod.rs | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 964f2ec5..f634be16 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -145,8 +145,7 @@ pub async fn generate_manifest_rusty( let total_manifest_length = Arc::new(AtomicU64::new(0)); // SAFETY: we .join_all() the futures using this - let backend: &'static (dyn VersionBackend + Send + Sync) = - unsafe { mem::transmute(&*backend) }; + let backend: &'static (dyn VersionBackend + Send + Sync) = unsafe { mem::transmute(&*backend) }; let mut futures: JoinSet> = JoinSet::new(); let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 6f3023c8..89874d1b 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -51,9 +51,7 @@ pub fn create_backend_constructor<'a>( let file_extension = path.extension().and_then(|v| v.to_str())?; - if SUPPORTED_FILE_EXTENSIONS - .contains(&file_extension) - { + if SUPPORTED_FILE_EXTENSIONS.contains(&file_extension) { let buf = path.to_path_buf(); return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); } From e5fd8f88774cf565bb63f0e0c9566bebfe71384b Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:31:37 +1100 Subject: [PATCH 49/50] fix: update tests with fail condition --- libraries/droplet/src/tests.rs | 20 ++++++++---------- .../testfiles/bare_files/emoji_filename.7z | Bin 347 -> 0 bytes .../testfiles/bare_files/unicode_filename.7z | Bin 347 -> 0 bytes .../subdirectories/empty_dotsubdirectory.7z | Bin 106 -> 0 bytes .../subdirectories/empty_subdirectory.7z | Bin 98 -> 0 bytes .../subdirectory_with_backslashes.7z | Bin 130 -> 0 bytes .../subdirectory_with_double_backslashes.7z | Bin 138 -> 0 bytes .../subdirectory_with_symlink_to_self.7z | Bin 263 -> 0 bytes 8 files changed, 9 insertions(+), 11 deletions(-) delete mode 100644 libraries/droplet/testfiles/bare_files/emoji_filename.7z delete mode 100644 libraries/droplet/testfiles/bare_files/unicode_filename.7z delete mode 100644 libraries/droplet/testfiles/subdirectories/empty_dotsubdirectory.7z delete mode 100644 libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z delete mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_backslashes.7z delete mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z delete mode 100644 libraries/droplet/testfiles/subdirectories/subdirectory_with_symlink_to_self.7z diff --git a/libraries/droplet/src/tests.rs b/libraries/droplet/src/tests.rs index 2b69c70c..0b81dd15 100644 --- a/libraries/droplet/src/tests.rs +++ b/libraries/droplet/src/tests.rs @@ -3,9 +3,7 @@ extern crate test_generator; use std::path::Path; -use serde_json::json; use test_generator::test_resources; -use tokio::{fs::File, io::AsyncWriteExt}; use crate::manifest::generate_manifest_rusty; @@ -35,14 +33,14 @@ fn manifest_gen(resource: &str) { ) }); - let mut output_path = filepath.to_path_buf(); - output_path.set_extension("json"); - - let mut file = File::create(output_path) - .await - .expect("failed to open output path"); - file.write_all(json!(manifest).to_string().as_bytes()) - .await - .expect("failed to write output"); + let first_chunk = manifest + .chunks + .values() + .next() + .expect("no chunks generated"); + let first_chunk_length = first_chunk.files.len(); + if first_chunk_length == 0 { + panic!("{} has no files in manifest", filepath.display()); + } }); } diff --git a/libraries/droplet/testfiles/bare_files/emoji_filename.7z b/libraries/droplet/testfiles/bare_files/emoji_filename.7z deleted file mode 100644 index f65e7d1e5a5471455caa6c0423532b2028b932a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 347 zcmXr7+Ou9=hJi)qOvaao3=j|nr7aF$W_bVqKf}#f28R5<|Nk%E{@>lsL$t^;F>uEu zjc1G7*nhrzBG+_T|EQG>qxZ(TlU-fD`ipsQ)-QY9qbwNwAX4IJY1gCjAKRwX^JE>k zSK^uS=w(dsCYBrDU3gZ<^X9s}e0sR7WBau5HQRh2X3FOO_U;hKaXQ^$AH&dp!Ee2q z#*{*N+kTaY1@o3G$G_1y-Fk8AjIKJZiJ#OJ?5=Yg)g@vg+g_#xMeKW7wkGn)(`Z(XpT}>l{qs9W>-&?b zstgQ_ENl#noDB~d*tr=Q7!?&61$Zt1{m;S0DD%otY#Lc@|q1_l655Q#JZ diff --git a/libraries/droplet/testfiles/bare_files/unicode_filename.7z b/libraries/droplet/testfiles/bare_files/unicode_filename.7z deleted file mode 100644 index 57999040802b169095d4a739ebd27391bddfdc91..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 347 zcmXr7+Ou9=hJoeugU=TpGC)8Sl%61bYr^~g{~2z^GBD)-{r`XQ_W$m79->8-iGe#N zX*^rp#{To&6S=0#`bVv77`->vo$TuJ)nCkevwqp*9%aGc2ays-OS>MG|JXLAo+s zG^P~F+xDwGESR@kIsT2t>DG%=XLQwRP5h)@`d#t>+YH|se{Zb6I)VR)YVeP*6U_`+ zmu`5Mf3U#Uo#*02!BuM$WOtq0s4fu`+4eFmC}Q8svNe%Uo<_5B{5*bZ?VsO4THl{c zRb^mcWMN}qWMpOJV{DLQ1q%r(Gw3m-Fyu3oFcdSCG9&@n rnG8h?sSL?Lk$fPl5~xg)L4=EuLC-eV^YqF)jG}Ce3<3>r9T^w^Nk9~K diff --git a/libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z b/libraries/droplet/testfiles/subdirectories/empty_subdirectory.7z deleted file mode 100644 index c58de4c1a472735a5e8f020fd824eafa3a6aeb87..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 98 zcmXr7+Ou9=hJhvWoAgu$C~$((Ps~=-FtRf8F*ZoDf`tU78HyQ78Il-M7%~}(7*ZLM m8A=%PfvifPv?PNF7bAmR(o50PEAKFhvN19UG`w|WU;qGx6cXbA diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_with_backslashes.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_with_backslashes.7z deleted file mode 100644 index 44f867e9343da299de22c9497a3c90616bbac38f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 130 zcmXr7+Ou9=hJj_S!{h@DP>=+rZ@*g7$;isc$Jijr3KkNyWhiDSWk_O3VaQ}CVn}63 zW+-9E2eK*|Vi?MSVkHb2Kvoh%B2Xq9s5S@4Dh7(B0!ao25iUjsJ=Vin1{> L2sFHPWMBXQu^t+1 diff --git a/libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z b/libraries/droplet/testfiles/subdirectories/subdirectory_with_double_backslashes.7z deleted file mode 100644 index a0ee82a6b7f69ade57d7e275eeb85a42336e9494..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 138 zcmXr7+Ou9=hJnS&u*Zb~3bLT|D&c?27+D$l7#k#6!9s$L48;tk3`qnn8y#*sOZa| z_a@cf;^1gdI{anN!H{K6iSwQ)7@yCZbLD5>BBn^~t5+BwPZPe(bLl|OmA#Xib0ad^ zk~d4N%V)4H4RA0FH2ePci=a@h-upF9Or;#s3HMcH4j$a$SF=cLvh38~GVkN1FSQ-$ zd(ZcHPuk6V+jj1x+x54f)_?Klbd-F5uG^kPDf7DepA6fnG1}lSU;nk?M z3$|6g`)nBJxvb)eHg|c^g4X%Zg%TJT#Mu}aIU9~LuyZpqFe)=MGP1@pI505qY@Wo$ LXm{YlaRvqeEl+G^ From 5f57fa8b4b5f3e19bda51b5313700bf172022280 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 2 Mar 2026 22:53:45 +1100 Subject: [PATCH 50/50] fix: panic on readbuf too small for block --- libraries/droplet/Cargo.lock | 2 +- libraries/droplet/Cargo.toml | 2 +- libraries/droplet/src/manifest.rs | 9 ++-- .../droplet/src/versions/archive_backend.rs | 41 ++++++++++++++++--- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index cf7b8dac..2a1665cf 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 0417f24b..5db83ec1 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.16.2" +version = "0.16.3" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index f634be16..f4c80138 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -156,7 +156,7 @@ pub async fn generate_manifest_rusty( let manifest = manifest.clone(); let reader_semaphore = reader_semaphore.clone(); futures.spawn(async move { - let mut read_buf = vec![0; 1024 * 1024 * 64]; + let mut read_buf = vec![0u8; 1024 * 1024 * 8]; let uuid = uuid::Uuid::new_v4().to_string(); let mut hasher = Sha256::new(); @@ -189,9 +189,10 @@ pub async fn generate_manifest_rusty( } total += amount; hasher.update(&read_buf[0..amount]); - if total as u64 > length { - panic!("read too much: target {}, got {}", length, total); - } + } + + if total as u64 > length { + panic!("read too much: target {}, got {}", length, total); } chunk_length += length; diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs index 75f84de6..2c0b4f22 100644 --- a/libraries/droplet/src/versions/archive_backend.rs +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -28,27 +28,53 @@ impl ZipVersionBackend { } } -struct ArchiveReader { +struct ArchiveReader<'a> { archive: FileReader, + prev_block: Option<&'a [u8]>, } -impl AsyncRead for ArchiveReader { +impl<'a> AsyncRead for ArchiveReader<'a> { fn poll_read( - self: std::pin::Pin<&mut Self>, + mut self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { + if let Some(block) = &mut self.prev_block { + let to_read = buf.remaining().min(block.len()); + let result = block.split_off(..to_read); + let result = result.unwrap(); // SAFETY: above .min statement + buf.put_slice(result); + + // If the block is empty, we can read more + if block.is_empty() { + self.prev_block = None; + } else { + return Poll::Ready(Ok(())); + } + } let block = match self.archive.read_block() { Ok(v) => v, Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))), }; - let block = match block { + let mut block = match block { Some(v) => v, None => return Poll::Ready(Ok(())), }; - buf.put_slice(block); + let write_amount = buf.remaining().min(block.len()); + let to_write = block.split_off(..write_amount); + let to_write = to_write.unwrap(); // SAFETY: above .min statement + buf.put_slice(to_write); + + if !block.is_empty() { + #[cfg(debug_assertions)] + if self.prev_block.is_some() { + panic!("replacing prev_block while it contains data") + } + self.prev_block.replace(&block[buf.remaining()..]); + } + Poll::Ready(Ok(())) } } @@ -95,7 +121,10 @@ impl VersionBackend for ZipVersionBackend { } } - Ok(Box::new(ArchiveReader { archive })) + Ok(Box::new(ArchiveReader { + archive, + prev_block: None, + })) } async fn peek_file(&self, sub_path: String) -> anyhow::Result {