From 7fabc826c70b9e034cda45f4871447a70bea9804 Mon Sep 17 00:00:00 2001 From: Jeremy Baxter Date: Wed, 2 Oct 2024 18:03:28 +1300 Subject: [PATCH] initial commit --- .editorconfig | 11 + .gitignore | 3 + COPYING | 674 +++ Makefile | 25 + arsd/cgi.d | 12525 ++++++++++++++++++++++++++++++++++++++++++++ arsd/core.d | 8581 ++++++++++++++++++++++++++++++ arsd/dom.d | 8778 +++++++++++++++++++++++++++++++ arsd/jsvar.d | 3039 +++++++++++ bluetop/server.d | 92 + buildconf.d | 12 + configure | 144 + static/index.html | 45 + 12 files changed, 33929 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 COPYING create mode 100644 Makefile create mode 100644 arsd/cgi.d create mode 100644 arsd/core.d create mode 100644 arsd/dom.d create mode 100644 arsd/jsvar.d create mode 100644 bluetop/server.d create mode 100644 buildconf.d create mode 100755 configure create mode 100644 static/index.html diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0163b4e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +indent_style = tab +indent_size = 4 +insert_final_newline = true + +[*.html] +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..127acb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.o +bluetopd +config.mk diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..553f8fe --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +all: bluetopd + +include config.mk + +PREFIX = /usr/local + +DC = ${_DC} +CFLAGS = ${_CFLAGS} -wi +OBJS = ${_OBJS} + +bluetopd: ${OBJS} + ${DC} ${_LDFLAGS} -of=$@ ${OBJS} + +.SUFFIXES: .d .o + +.d.o: + ${DC} ${CFLAGS} -of=$@ -c $< + +clean: + rm -f bluetopd ${OBJS} + +install: bluetopd + install -Dm755 bluetopd ${DESTDIR}${PREFIX}/bin/bluetopd + +.PHONY: all clean install diff --git a/arsd/cgi.d b/arsd/cgi.d new file mode 100644 index 0000000..9cb31f1 --- /dev/null +++ b/arsd/cgi.d @@ -0,0 +1,12525 @@ +// FIXME: if an exception is thrown, we shouldn't necessarily cache... +// FIXME: there's some annoying duplication of code in the various versioned mains + +// add the Range header in there too. should return 206 + +// FIXME: cgi per-request arena allocator + +// i need to add a bunch of type templates for validations... mayne @NotNull or NotNull! + +// FIXME: I might make a cgi proxy class which can change things; the underlying one is still immutable +// but the later one can edit and simplify the api. You'd have to use the subclass tho! + +/* +void foo(int f, @("test") string s) {} + +void main() { + static if(is(typeof(foo) Params == __parameters)) + //pragma(msg, __traits(getAttributes, Params[0])); + pragma(msg, __traits(getAttributes, Params[1..2])); + else + pragma(msg, "fail"); +} +*/ + +// Note: spawn-fcgi can help with fastcgi on nginx + +// FIXME: to do: add openssl optionally +// make sure embedded_httpd doesn't send two answers if one writes() then dies + +// future direction: websocket as a separate process that you can sendfile to for an async passoff of those long-lived connections + +/* + Session manager process: it spawns a new process, passing a + command line argument, to just be a little key/value store + of some serializable struct. On Windows, it CreateProcess. + On Linux, it can just fork or maybe fork/exec. The session + key is in a cookie. + + Server-side event process: spawns an async manager. You can + push stuff out to channel ids and the clients listen to it. + + websocket process: spawns an async handler. They can talk to + each other or get info from a cgi request. + + Tempting to put web.d 2.0 in here. It would: + * map urls and form generation to functions + * have data presentation magic + * do the skeleton stuff like 1.0 + * auto-cache generated stuff in files (at least if pure?) + * introspect functions in json for consumers + + + https://linux.die.net/man/3/posix_spawn +*/ + +/++ + Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications. Offers both lower- and higher- level api options among other common (optional) things like websocket and event source serving support, session management, and job scheduling. + + --- + import arsd.cgi; + + // Instead of writing your own main(), you should write a function + // that takes a Cgi param, and use mixin GenericMain + // for maximum compatibility with different web servers. + void hello(Cgi cgi) { + cgi.setResponseContentType("text/plain"); + + if("name" in cgi.get) + cgi.write("Hello, " ~ cgi.get["name"]); + else + cgi.write("Hello, world!"); + } + + mixin GenericMain!hello; + --- + + Or: + --- + import arsd.cgi; + + class MyApi : WebObject { + @UrlName("") + string hello(string name = null) { + if(name is null) + return "Hello, world!"; + else + return "Hello, " ~ name; + } + } + mixin DispatcherMain!( + "/".serveApi!MyApi + ); + --- + + $(NOTE + Please note that using the higher-level api will add a dependency on arsd.dom and arsd.jsvar to your application. + If you use `dmd -i` or `ldc2 -i` to build, it will just work, but with dub, you will have do `dub add arsd-official:jsvar` + and `dub add arsd-official:dom` yourself. + ) + + Test on console (works in any interface mode): + $(CONSOLE + $ ./cgi_hello GET / name=whatever + ) + + If using http version (default on `dub` builds, or on custom builds when passing `-version=embedded_httpd` to dmd): + $(CONSOLE + $ ./cgi_hello --port 8080 + # now you can go to http://localhost:8080/?name=whatever + ) + + Please note: the default port for http is 8085 and for scgi is 4000. I recommend you set your own by the command line argument in a startup script instead of relying on any hard coded defaults. It is possible though to code your own with [RequestServer], however. + + + Build_Configurations: + + cgi.d tries to be flexible to meet your needs. It is possible to configure it both at runtime (by writing your own `main` function and constructing a [RequestServer] object) or at compile time using the `version` switch to the compiler or a dub `subConfiguration`. + + If you are using `dub`, use: + + ```sdlang + subConfiguration "arsd-official:cgi" "VALUE_HERE" + ``` + + or to dub.json: + + ```json + "subConfigurations": {"arsd-official:cgi": "VALUE_HERE"} + ``` + + to change versions. The possible options for `VALUE_HERE` are: + + $(LIST + * `embedded_httpd` for the embedded httpd version (built-in web server). This is the default for dub builds. You can run the program then connect directly to it from your browser. Note: prior to version 11, this would be embedded_httpd_processes on Linux and embedded_httpd_threads everywhere else. It now means embedded_httpd_hybrid everywhere supported and embedded_httpd_threads everywhere else. + * `cgi` for traditional cgi binaries. These are run by an outside web server as-needed to handle requests. + * `fastcgi` for FastCGI builds. FastCGI is managed from an outside helper, there's one built into Microsoft IIS, Apache httpd, and Lighttpd, and a generic program you can use with nginx called `spawn-fcgi`. If you don't already know how to use it, I suggest you use one of the other modes. + * `scgi` for SCGI builds. SCGI is a simplified form of FastCGI, where you run the server as an application service which is proxied by your outside webserver. Please note: on nginx make sure you add `scgi_param PATH_INFO $document_uri;` to the config! + * `stdio_http` for speaking raw http over stdin and stdout. This is made for systemd services. See [RequestServer.serveSingleHttpConnectionOnStdio] for more information. + ) + + With dmd, use: + + $(TABLE_ROWS + + * + Interfaces + + (mutually exclusive) + + * - `-version=plain_cgi` + - The default building the module alone without dub - a traditional, plain CGI executable will be generated. + * - `-version=embedded_httpd` + - A HTTP server will be embedded in the generated executable. This is default when building with dub. + * - `-version=fastcgi` + - A FastCGI executable will be generated. + * - `-version=scgi` + - A SCGI (SimpleCGI) executable will be generated. + * - `-version=embedded_httpd_hybrid` + - A HTTP server that uses a combination of processes, threads, and fibers to better handle large numbers of idle connections. Recommended if you are going to serve websockets in a non-local application. + * - `-version=embedded_httpd_threads` + - The embedded HTTP server will use a single process with a thread pool. (use instead of plain `embedded_httpd` if you want this specific implementation) + * - `-version=embedded_httpd_processes` + - The embedded HTTP server will use a prefork style process pool. (use instead of plain `embedded_httpd` if you want this specific implementation) + * - `-version=embedded_httpd_processes_accept_after_fork` + - It will call accept() in each child process, after forking. This is currently the only option, though I am experimenting with other ideas. You probably should NOT specify this right now. + * - `-version=stdio_http` + - The embedded HTTP server will be spoken over stdin and stdout. + + * + Tweaks + + (can be used together with others) + + * - `-version=cgi_with_websocket` + - The CGI class has websocket server support. (This is on by default now.) + + * - `-version=with_openssl` + - not currently used + * - `-version=cgi_embedded_sessions` + - The session server will be embedded in the cgi.d server process + * - `-version=cgi_session_server_process` + - The session will be provided in a separate process, provided by cgi.d. + ) + + For example, + + For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory. + + For FastCGI: `dmd yourfile.d cgi.d -version=fastcgi` and run it. spawn-fcgi helps on nginx. You can put the file in the directory for Apache. On IIS, run it with a port on the command line (this causes it to call FCGX_OpenSocket, which can work on nginx too). + + For SCGI: `dmd yourfile.d cgi.d -version=scgi` and run the executable, providing a port number on the command line. + + For an embedded HTTP server, run `dmd yourfile.d cgi.d -version=embedded_httpd` and run the generated program. It listens on port 8085 by default. You can change this on the command line with the --port option when running your program. + + Simulating_requests: + + If you are using one of the [GenericMain] or [DispatcherMain] mixins, or main with your own call to [RequestServer.trySimulatedRequest], you can simulate requests from your command-ine shell. Call the program like this: + + $(CONSOLE + ./yourprogram GET / name=adr + ) + + And it will print the result to stdout instead of running a server, regardless of build more.. + + CGI_Setup_tips: + + On Apache, you may do `SetHandler cgi-script` in your `.htaccess` file to set a particular file to be run through the cgi program. Note that all "subdirectories" of it also run the program; if you configure `/foo` to be a cgi script, then going to `/foo/bar` will call your cgi handler function with `cgi.pathInfo == "/bar"`. + + Overview_Of_Basic_Concepts: + + cgi.d offers both lower-level handler apis as well as higher-level auto-dispatcher apis. For a lower-level handler function, you'll probably want to review the following functions: + + Input: [Cgi.get], [Cgi.post], [Cgi.request], [Cgi.files], [Cgi.cookies], [Cgi.pathInfo], [Cgi.requestMethod], + and HTTP headers ([Cgi.headers], [Cgi.userAgent], [Cgi.referrer], [Cgi.accept], [Cgi.authorization], [Cgi.lastEventId]) + + Output: [Cgi.write], [Cgi.header], [Cgi.setResponseStatus], [Cgi.setResponseContentType], [Cgi.gzipResponse] + + Cookies: [Cgi.setCookie], [Cgi.clearCookie], [Cgi.cookie], [Cgi.cookies] + + Caching: [Cgi.setResponseExpires], [Cgi.updateResponseExpires], [Cgi.setCache] + + Redirections: [Cgi.setResponseLocation] + + Other Information: [Cgi.remoteAddress], [Cgi.https], [Cgi.port], [Cgi.scriptName], [Cgi.requestUri], [Cgi.getCurrentCompleteUri], [Cgi.onRequestBodyDataReceived] + + Websockets: [Websocket], [websocketRequested], [acceptWebsocket]. For websockets, use the `embedded_httpd_hybrid` build mode for best results, because it is optimized for handling large numbers of idle connections compared to the other build modes. + + Overriding behavior for special cases streaming input data: see the virtual functions [Cgi.handleIncomingDataChunk], [Cgi.prepareForIncomingDataChunks], [Cgi.cleanUpPostDataState] + + A basic program using the lower-level api might look like: + + --- + import arsd.cgi; + + // you write a request handler which always takes a Cgi object + void handler(Cgi cgi) { + /+ + when the user goes to your site, suppose you are being hosted at http://example.com/yourapp + + If the user goes to http://example.com/yourapp/test?name=value + then the url will be parsed out into the following pieces: + + cgi.pathInfo == "/test". This is everything after yourapp's name. (If you are doing an embedded http server, your app's name is blank, so pathInfo will be the whole path of the url.) + + cgi.scriptName == "yourapp". With an embedded http server, this will be blank. + + cgi.host == "example.com" + + cgi.https == false + + cgi.queryString == "name=value" (there's also cgi.search, which will be "?name=value", including the ?) + + The query string is further parsed into the `get` and `getArray` members, so: + + cgi.get == ["name": "value"], meaning you can do `cgi.get["name"] == "value"` + + And + + cgi.getArray == ["name": ["value"]]. + + Why is there both `get` and `getArray`? The standard allows names to be repeated. This can be very useful, + it is how http forms naturally pass multiple items like a set of checkboxes. So `getArray` is the complete data + if you need it. But since so often you only care about one value, the `get` member provides more convenient access. + + We can use these members to process the request and build link urls. Other info from the request are in other members, we'll look at them later. + +/ + switch(cgi.pathInfo) { + // the home page will be a small html form that can set a cookie. + case "/": + cgi.write(` + + +
+ + +
+ + + `, true); // the , true tells it that this is the one, complete response i want to send, allowing some optimizations. + break; + // POSTing to this will set a cookie with our submitted name + case "/set-cookie": + // HTTP has a number of request methods (also called "verbs") to tell + // what you should do with the given resource. + // The most common are GET and POST, the ones used in html forms. + // You can check which one was used with the `cgi.requestMethod` property. + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + + // headers like redirections need to be set before we call `write` + cgi.setResponseLocation("read-cookie"); + + // just like how url params go into cgi.get/getArray, form data submitted in a POST + // body go to cgi.post/postArray. Please note that a POST request can also have get + // params in addition to post params. + // + // There's also a convenience function `cgi.request("name")` which checks post first, + // then get if it isn't found there, and then returns a default value if it is in neither. + if("name" in cgi.post) { + // we can set cookies with a method too + // again, cookies need to be set before calling `cgi.write`, since they + // are a kind of header. + cgi.setCookie("name" , cgi.post["name"]); + } + + // the user will probably never see this, since the response location + // is an automatic redirect, but it is still best to say something anyway + cgi.write("Redirecting you to see the cookie...", true); + } else { + // you can write out response codes and headers + // as well as response bodies + // + // But always check the cgi docs before using the generic + // `header` method - if there is a specific method for your + // header, use it before resorting to the generic one to avoid + // a header value from being sent twice. + cgi.setResponseLocation("405 Method Not Allowed"); + // there is no special accept member, so you can use the generic header function + cgi.header("Accept: POST"); + // but content type does have a method, so prefer to use it: + cgi.setResponseContentType("text/plain"); + + // all the headers are buffered, and will be sent upon the first body + // write. you can actually modify some of them before sending if need be. + cgi.write("You must use the POST http verb on this resource.", true); + } + break; + // and GETting this will read the cookie back out + case "/read-cookie": + // I did NOT pass `,true` here because this is writing a partial response. + // It is possible to stream data to the user in chunks by writing partial + // responses the calling `cgi.flush();` to send the partial response immediately. + // normally, you'd only send partial chunks if you have to - it is better to build + // a response as a whole and send it as a whole whenever possible - but here I want + // to demo that you can. + cgi.write("Hello, "); + if("name" in cgi.cookies) { + import arsd.dom; // dom.d provides a lot of helpers for html + // since the cookie is set, we need to write it out properly to + // avoid cross-site scripting attacks. + // + // Getting this stuff right automatically is a benefit of using the higher + // level apis, but this demo is to show the fundamental building blocks, so + // we're responsible to take care of it. + cgi.write(htmlEntitiesEncode(cgi.cookies["name"])); + } else { + cgi.write("friend"); + } + + // note that I never called cgi.setResponseContentType, since the default is text/html. + // it doesn't hurt to do it explicitly though, just remember to do it before any cgi.write + // calls. + break; + default: + // no path matched + cgi.setResponseStatus("404 Not Found"); + cgi.write("Resource not found.", true); + } + } + + // and this adds the boilerplate to set up a server according to the + // compile version configuration and call your handler as requests come in + mixin GenericMain!handler; // the `handler` here is the name of your function + --- + + Even if you plan to always use the higher-level apis, I still recommend you at least familiarize yourself with the lower level functions, since they provide the lightest weight, most flexible options to get down to business if you ever need them. + + In the lower-level api, the [Cgi] object represents your HTTP transaction. It has functions to describe the request and for you to send your response. It leaves the details of how you o it up to you. The general guideline though is to avoid depending any variables outside your handler function, since there's no guarantee they will survive to another handler. You can use global vars as a lazy initialized cache, but you should always be ready in case it is empty. (One exception: if you use `-version=embedded_httpd_threads -version=cgi_no_fork`, then you can rely on it more, but you should still really write things assuming your function won't have anything survive beyond its return for max scalability and compatibility.) + + A basic program using the higher-level apis might look like: + + --- + /+ + import arsd.cgi; + + struct LoginData { + string currentUser; + } + + class AppClass : WebObject { + string foo() {} + } + + mixin DispatcherMain!( + "/assets/.serveStaticFileDirectory("assets/", true), // serve the files in the assets subdirectory + "/".serveApi!AppClass, + "/thing/".serveRestObject, + ); + +/ + --- + + Guide_for_PHP_users: + (Please note: I wrote this section in 2008. A lot of PHP hosts still ran 4.x back then, so it was common to avoid using classes - introduced in php 5 - to maintain compatibility! If you're coming from php more recently, this may not be relevant anymore, but still might help you.) + + If you are coming from old-style PHP, here's a quick guide to help you get started: + + $(SIDE_BY_SIDE + $(COLUMN + ```php + + ``` + ) + $(COLUMN + --- + import arsd.cgi; + void app(Cgi cgi) { + string foo = cgi.post["foo"]; + string bar = cgi.get["bar"]; + string baz = cgi.cookies["baz"]; + + string user_ip = cgi.remoteAddress; + string host = cgi.host; + string path = cgi.pathInfo; + + cgi.setCookie("baz", "some value"); + + cgi.write("hello!"); + } + + mixin GenericMain!app + --- + ) + ) + + $(H3 Array elements) + + + In PHP, you can give a form element a name like `"something[]"`, and then + `$_POST["something"]` gives an array. In D, you can use whatever name + you want, and access an array of values with the `cgi.getArray["name"]` and + `cgi.postArray["name"]` members. + + $(H3 Databases) + + PHP has a lot of stuff in its standard library. cgi.d doesn't include most + of these, but the rest of my arsd repository has much of it. For example, + to access a MySQL database, download `database.d` and `mysql.d` from my + github repo, and try this code (assuming, of course, your database is + set up): + + --- + import arsd.cgi; + import arsd.mysql; + + void app(Cgi cgi) { + auto database = new MySql("localhost", "username", "password", "database_name"); + foreach(row; mysql.query("SELECT count(id) FROM people")) + cgi.write(row[0] ~ " people in database"); + } + + mixin GenericMain!app; + --- + + Similar modules are available for PostgreSQL, Microsoft SQL Server, and SQLite databases, + implementing the same basic interface. + + See_Also: + + You may also want to see [arsd.dom], [arsd.webtemplate], and maybe some functions from my old [arsd.html] for more code for making + web applications. dom and webtemplate are used by the higher-level api here in cgi.d. + + For working with json, try [arsd.jsvar]. + + [arsd.database], [arsd.mysql], [arsd.postgres], [arsd.mssql], and [arsd.sqlite] can help in + accessing databases. + + If you are looking to access a web application via HTTP, try [arsd.http2]. + + Copyright: + + cgi.d copyright 2008-2023, Adam D. Ruppe. Provided under the Boost Software License. + + Yes, this file is old, and yes, it is still actively maintained and used. + + History: + An import of `arsd.core` was added on March 21, 2023 (dub v11.0). Prior to this, the module's default configuration was completely stand-alone. You must now include the `core.d` file in your builds with `cgi.d`. + + This change is primarily to integrate the event loops across the library, allowing you to more easily use cgi.d along with my other libraries like simpledisplay and http2.d. Previously, you'd have to run separate helper threads. Now, they can all automatically work together. ++/ +module arsd.cgi; + +/* bluetop */ version = embedded_httpd; + +// FIXME: Nullable!T can be a checkbox that enables/disables the T on the automatic form +// and a SumType!(T, R) can be a radio box to pick between T and R to disclose the extra boxes on the automatic form + +/++ + This micro-example uses the [dispatcher] api to act as a simple http file server, serving files found in the current directory and its children. ++/ +version(Demo) +unittest { + import arsd.cgi; + + mixin DispatcherMain!( + "/".serveStaticFileDirectory(null, true) + ); +} + +/++ + Same as the previous example, but written out long-form without the use of [DispatcherMain] nor [GenericMain]. ++/ +version(Demo) +unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + cgi.dispatcher!( + "/".serveStaticFileDirectory(null, true) + ); + } + + // mixin GenericMain!requestHandler would add this function: + void main(string[] args) { + // this is all the content of [cgiMainImpl] which you can also call + + // cgi.d embeds a few add on functions like real time event forwarders + // and session servers it can run in other processes. this spawns them, if needed. + if(tryAddonServers(args)) + return; + + // cgi.d allows you to easily simulate http requests from the command line, + // without actually starting a server. this function will do that. + if(trySimulatedRequest!(requestHandler, Cgi)(args)) + return; + + RequestServer server; + // you can change the default port here if you like + // server.listeningPort = 9000; + + // then call this to let the command line args override your default + server.configureFromCommandLine(args); + + // here is where you could print out the listeningPort to the user if you wanted + + // and serve the request(s) according to the compile configuration + server.serve!(requestHandler)(); + + // or you could explicitly choose a serve mode like this: + // server.serveEmbeddedHttp!requestHandler(); + } +} + +/++ + cgi.d has built-in testing helpers too. These will provide mock requests and mock sessions that + otherwise run through the rest of the internal mechanisms to call your functions without actually + spinning up a server. ++/ +version(Demo) +unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + + } + + // D doesn't let me embed a unittest inside an example unittest + // so this is a function, but you can do it however in your real program + /* unittest */ void runTests() { + auto tester = new CgiTester(&requestHandler); + + auto response = tester.GET("/"); + assert(response.code == 200); + } +} + +/++ + The session system works via a built-in spawnable server. + + Bugs: + Requires addon servers, which are not implemented yet on Windows. ++/ +version(Posix) +version(Demo) +unittest { + import arsd.cgi; + + struct SessionData { + string userId; + } + + void handler(Cgi cgi) { + auto session = cgi.getSessionObject!SessionData; + + if(cgi.pathInfo == "/login") { + session.userId = cgi.queryString; + cgi.setResponseLocation("view"); + } else { + cgi.write(session.userId); + } + } + + mixin GenericMain!handler; +} + +static import std.file; + +static import arsd.core; +version(Posix) +import arsd.core : makeNonBlocking; + +import arsd.core : encodeUriComponent, decodeUriComponent; + + +// for a single thread, linear request thing, use: +// -version=embedded_httpd_threads -version=cgi_no_threads + +version(Posix) { + version(CRuntime_Musl) { + + } else version(minimal) { + + } else { + version(FreeBSD) { + // I never implemented the fancy stuff there either + } else { + version=with_breaking_cgi_features; + version=with_sendfd; + version=with_addon_servers; + } + } +} + +version(Windows) { + version(minimal) { + + } else { + // not too concerned about gdc here since the mingw version is fairly new as well + version=with_breaking_cgi_features; + } +} + +// FIXME: can use the arsd.core function now but it is trivial anyway tbh +void cloexec(int fd) { + version(Posix) { + import core.sys.posix.fcntl; + fcntl(fd, F_SETFD, FD_CLOEXEC); + } +} + +void cloexec(Socket s) { + version(Posix) { + import core.sys.posix.fcntl; + fcntl(s.handle, F_SETFD, FD_CLOEXEC); + } +} + +// the servers must know about the connections to talk to them; the interfaces are vital +version(with_addon_servers) + version=with_addon_servers_connections; + +version(embedded_httpd) { + version(OSX) + version = embedded_httpd_threads; + else + version=embedded_httpd_hybrid; + /* + version(with_openssl) { + pragma(lib, "crypto"); + pragma(lib, "ssl"); + } + */ +} + +version(embedded_httpd_hybrid) { + version=embedded_httpd_threads; + version(cgi_no_fork) {} else version(Posix) + version=cgi_use_fork; + version=cgi_use_fiber; +} + +version(cgi_use_fork) + enum cgi_use_fork_default = true; +else + enum cgi_use_fork_default = false; + +version(embedded_httpd_processes) + version=embedded_httpd_processes_accept_after_fork; // I am getting much better average performance on this, so just keeping it. But the other way MIGHT help keep the variation down so i wanna keep the code to play with later + +version(embedded_httpd_threads) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} +version(scgi) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} + +// fall back if the other is not defined so we can cleanly version it below +version(cgi_embedded_sessions) {} +else version=cgi_session_server_process; + + +version=cgi_with_websocket; + +enum long defaultMaxContentLength = 5_000_000; + +/* + + To do a file download offer in the browser: + + cgi.setResponseContentType("text/csv"); + cgi.header("Content-Disposition: attachment; filename=\"customers.csv\""); +*/ + +// FIXME: the location header is supposed to be an absolute url I guess. + +// FIXME: would be cool to flush part of a dom document before complete +// somehow in here and dom.d. + + +// these are public so you can mixin GenericMain. +// FIXME: use a function level import instead! +public import std.string; +public import std.stdio; +public import std.conv; +import std.uni; +import std.algorithm.comparison; +import std.algorithm.searching; +import std.exception; +import std.base64; +static import std.algorithm; +import std.datetime; +import std.range; + +import std.process; + +import std.zlib; + + +T[] consume(T)(T[] range, int count) { + if(count > range.length) + count = range.length; + return range[count..$]; +} + +int locationOf(T)(T[] data, string item) { + const(ubyte[]) d = cast(const(ubyte[])) data; + const(ubyte[]) i = cast(const(ubyte[])) item; + + // this is a vague sanity check to ensure we aren't getting insanely + // sized input that will infinite loop below. it should never happen; + // even huge file uploads ought to come in smaller individual pieces. + if(d.length > (int.max/2)) + throw new Exception("excessive block of input"); + + for(int a = 0; a < d.length; a++) { + if(a + i.length > d.length) + return -1; + if(d[a..a+i.length] == i) + return a; + } + + return -1; +} + +/// If you are doing a custom cgi class, mixing this in can take care of +/// the required constructors for you +mixin template ForwardCgiConstructors() { + this(long maxContentLength = defaultMaxContentLength, + string[string] env = null, + const(ubyte)[] delegate() readdata = null, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null + ) { super(maxContentLength, env, readdata, _rawDataOutput, _flush); } + + this(string[] args) { super(args); } + + this( + BufferedInputRange inputData, + string address, ushort _port, + int pathInfoStarts = 0, + bool _https = false, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null, + // this pointer tells if the connection is supposed to be closed after we handle this + bool* closeConnection = null) + { + super(inputData, address, _port, pathInfoStarts, _https, _rawDataOutput, _flush, closeConnection); + } + + this(BufferedInputRange ir, bool* closeConnection) { super(ir, closeConnection); } +} + +/// thrown when a connection is closed remotely while we waiting on data from it +class ConnectionClosedException : Exception { + this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(message, file, line, next); + } +} + + +version(Windows) { +// FIXME: ugly hack to solve stdin exception problems on Windows: +// reading stdin results in StdioException (Bad file descriptor) +// this is probably due to http://d.puremagic.com/issues/show_bug.cgi?id=3425 +private struct stdin { + struct ByChunk { // Replicates std.stdio.ByChunk + private: + ubyte[] chunk_; + public: + this(size_t size) + in { + assert(size, "size must be larger than 0"); + } + do { + chunk_ = new ubyte[](size); + popFront(); + } + + @property bool empty() const { + return !std.stdio.stdin.isOpen || std.stdio.stdin.eof; // Ugly, but seems to do the job + } + @property nothrow ubyte[] front() { return chunk_; } + void popFront() { + enforce(!empty, "Cannot call popFront on empty range"); + chunk_ = stdin.rawRead(chunk_); + } + } + + import core.sys.windows.windows; +static: + + T[] rawRead(T)(T[] buf) { + uint bytesRead; + auto result = ReadFile(GetStdHandle(STD_INPUT_HANDLE), buf.ptr, cast(int) (buf.length * T.sizeof), &bytesRead, null); + + if (!result) { + auto err = GetLastError(); + if (err == 38/*ERROR_HANDLE_EOF*/ || err == 109/*ERROR_BROKEN_PIPE*/) // 'good' errors meaning end of input + return buf[0..0]; + // Some other error, throw it + + char* buffer; + scope(exit) LocalFree(buffer); + + // FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100 + // FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 + FormatMessageA(0x1100, null, err, 0, cast(char*)&buffer, 256, null); + throw new Exception(to!string(buffer)); + } + enforce(!(bytesRead % T.sizeof), "I/O error"); + return buf[0..bytesRead / T.sizeof]; + } + + auto byChunk(size_t sz) { return ByChunk(sz); } + + void close() { + std.stdio.stdin.close; + } +} +} + +/// The main interface with the web request +class Cgi { + public: + /// the methods a request can be + enum RequestMethod { GET, HEAD, POST, PUT, DELETE, // GET and POST are the ones that really work + // these are defined in the standard, but idk if they are useful for anything + OPTIONS, TRACE, CONNECT, + // These seem new, I have only recently seen them + PATCH, MERGE, + // this is an extension for when the method is not specified and you want to assume + CommandLine } + + + /+ + + ubyte[] perRequestMemoryPool; + void[] perRequestMemoryPoolWithPointers; + // might want to just slice the buffer itself too when we happened to have gotten a full request inside it and don't need to decode + // then the buffer also can be recycled if it is set. + + // we might also be able to set memory recyclable true by default, but then the property getters set it to false. but not all the things are property getters. but realistically anything except benchmarks are gonna get something lol so meh. + + /+ + struct VariableCollection { + string[] opIndex(string name) { + + } + } + + /++ + Call this to indicate that you've not retained any reference to the request-local memory (including all strings returned from the Cgi object) outside the request (you can .idup anything you need to store) and it is thus free to be freed or reused by another request. + + Most handlers should be able to call this; retaining memory is the exception in any cgi program, but since I can't prove it from inside the library, it plays it safe and lets the GC manage it unless you opt into this behavior. All cgi.d functions will duplicate strings if needed (e.g. session ids from cookies) so unless you're doing something yourself, this should be ok. + + History: + Added + +/ + public void recycleMemory() { + + } + +/ + + + /++ + Cgi provides a per-request memory pool + + +/ + void[] allocateMemory(size_t nBytes) { + + } + + /// ditto + void[] reallocateMemory(void[] old, size_t nBytes) { + + } + + /// ditto + void freeMemory(void[] memory) { + + } + +/ + + +/* + import core.runtime; + auto args = Runtime.args(); + + we can call the app a few ways: + + 1) set up the environment variables and call the app (manually simulating CGI) + 2) simulate a call automatically: + ./app method 'uri' + + for example: + ./app get /path?arg arg2=something + + Anything on the uri is treated as query string etc + + on get method, further args are appended to the query string (encoded automatically) + on post method, further args are done as post + + + @name means import from file "name". if name == -, it uses stdin + (so info=@- means set info to the value of stdin) + + + Other arguments include: + --cookie name=value (these are all concated together) + --header 'X-Something: cool' + --referrer 'something' + --port 80 + --remote-address some.ip.address.here + --https yes + --user-agent 'something' + --userpass 'user:pass' + --authorization 'Basic base64encoded_user:pass' + --accept 'content' // FIXME: better example + --last-event-id 'something' + --host 'something.com' + --session name=value (these are added to a mock session, changes to the session are printed out as dummy response headers) + + Non-simulation arguments: + --port xxx listening port for non-cgi things (valid for the cgi interfaces) + --listening-host the ip address the application should listen on, or if you want to use unix domain sockets, it is here you can set them: `--listening-host unix:filename` or, on Linux, `--listening-host abstract:name`. + +*/ + + /** Initializes it with command line arguments (for easy testing) */ + this(string[] args, void delegate(const(ubyte)[]) _rawDataOutput = null) { + rawDataOutput = _rawDataOutput; + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + + bool lookingForMethod; + bool lookingForUri; + string nextArgIs; + + string _cookie; + string _queryString; + string[][string] _post; + string[string] _headers; + + string[] breakUp(string s) { + string k, v; + auto idx = s.indexOf("="); + if(idx == -1) { + k = s; + } else { + k = s[0 .. idx]; + v = s[idx + 1 .. $]; + } + + return [k, v]; + } + + lookingForMethod = true; + + scriptName = args[0]; + scriptFileName = args[0]; + + environmentVariables = cast(const) environment.toAA; + + foreach(arg; args[1 .. $]) { + if(arg.startsWith("--")) { + nextArgIs = arg[2 .. $]; + } else if(nextArgIs.length) { + if (nextArgIs == "cookie") { + auto info = breakUp(arg); + if(_cookie.length) + _cookie ~= "; "; + _cookie ~= encodeUriComponent(info[0]) ~ "=" ~ encodeUriComponent(info[1]); + } + if (nextArgIs == "session") { + auto info = breakUp(arg); + _commandLineSession[info[0]] = info[1]; + } + + else if (nextArgIs == "port") { + port = to!int(arg); + } + else if (nextArgIs == "referrer") { + referrer = arg; + } + else if (nextArgIs == "remote-address") { + remoteAddress = arg; + } + else if (nextArgIs == "user-agent") { + userAgent = arg; + } + else if (nextArgIs == "authorization") { + authorization = arg; + } + else if (nextArgIs == "userpass") { + authorization = "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (arg)).idup; + } + else if (nextArgIs == "origin") { + origin = arg; + } + else if (nextArgIs == "accept") { + accept = arg; + } + else if (nextArgIs == "last-event-id") { + lastEventId = arg; + } + else if (nextArgIs == "https") { + if(arg == "yes") + https = true; + } + else if (nextArgIs == "header") { + string thing, other; + auto idx = arg.indexOf(":"); + if(idx == -1) + throw new Exception("need a colon in a http header"); + thing = arg[0 .. idx]; + other = arg[idx + 1.. $]; + _headers[thing.strip.toLower()] = other.strip; + } + else if (nextArgIs == "host") { + host = arg; + } + // else + // skip, we don't know it but that's ok, it might be used elsewhere so no error + + nextArgIs = null; + } else if(lookingForMethod) { + lookingForMethod = false; + lookingForUri = true; + + if(arg.asLowerCase().equal("commandline")) + requestMethod = RequestMethod.CommandLine; + else + requestMethod = to!RequestMethod(arg.toUpper()); + } else if(lookingForUri) { + lookingForUri = false; + + requestUri = arg; + + auto idx = arg.indexOf("?"); + if(idx == -1) + pathInfo = arg; + else { + pathInfo = arg[0 .. idx]; + _queryString = arg[idx + 1 .. $]; + } + } else { + // it is an argument of some sort + if(requestMethod == Cgi.RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT || requestMethod == Cgi.RequestMethod.CommandLine) { + auto parts = breakUp(arg); + _post[parts[0]] ~= parts[1]; + allPostNamesInOrder ~= parts[0]; + allPostValuesInOrder ~= parts[1]; + } else { + if(_queryString.length) + _queryString ~= "&"; + auto parts = breakUp(arg); + _queryString ~= encodeUriComponent(parts[0]) ~ "=" ~ encodeUriComponent(parts[1]); + } + } + } + + acceptsGzip = false; + keepAliveRequested = false; + requestHeaders = cast(immutable) _headers; + + cookie = _cookie; + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + queryString = _queryString; + getArray = cast(immutable) decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + get = keepLastOf(getArray); + + postArray = cast(immutable) _post; + post = keepLastOf(_post); + + // FIXME + filesArray = null; + files = null; + + isCalledWithCommandLineArguments = true; + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + this.postBody = null; + this.requestContentType = null; + } + + private { + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + string[] allGetNamesInOrder; + string[] allGetValuesInOrder; + } + + CgiConnectionHandle getOutputFileHandle() { + return _outputFileHandle; + } + + CgiConnectionHandle _outputFileHandle = INVALID_CGI_CONNECTION_HANDLE; + + /** Initializes it using a CGI or CGI-like interface */ + this(long maxContentLength = defaultMaxContentLength, + // use this to override the environment variable listing + in string[string] env = null, + // and this should return a chunk of data. return empty when done + const(ubyte)[] delegate() readdata = null, + // finally, use this to do custom output if needed + void delegate(const(ubyte)[]) _rawDataOutput = null, + // to flush teh custom output + void delegate() _flush = null + ) + { + + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + + + + isCalledWithCommandLineArguments = false; + rawDataOutput = _rawDataOutput; + flushDelegate = _flush; + auto getenv = delegate string(string var) { + if(env is null) + return std.process.environment.get(var); + auto e = var in env; + if(e is null) + return null; + return *e; + }; + + environmentVariables = env is null ? + cast(const) environment.toAA : + env; + + // fetching all the request headers + string[string] requestHeadersHere; + foreach(k, v; env is null ? cast(const) environment.toAA() : env) { + if(k.startsWith("HTTP_")) { + requestHeadersHere[replace(k["HTTP_".length .. $].toLower(), "_", "-")] = v; + } + } + + this.requestHeaders = assumeUnique(requestHeadersHere); + + requestUri = getenv("REQUEST_URI"); + + cookie = getenv("HTTP_COOKIE"); + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + referrer = getenv("HTTP_REFERER"); + userAgent = getenv("HTTP_USER_AGENT"); + remoteAddress = getenv("REMOTE_ADDR"); + host = getenv("HTTP_HOST"); + pathInfo = getenv("PATH_INFO"); + + queryString = getenv("QUERY_STRING"); + scriptName = getenv("SCRIPT_NAME"); + { + import core.runtime; + auto sfn = getenv("SCRIPT_FILENAME"); + scriptFileName = sfn.length ? sfn : (Runtime.args.length ? Runtime.args[0] : null); + } + + bool iis = false; + + // Because IIS doesn't pass requestUri, we simulate it here if it's empty. + if(requestUri.length == 0) { + // IIS sometimes includes the script name as part of the path info - we don't want that + if(pathInfo.length >= scriptName.length && (pathInfo[0 .. scriptName.length] == scriptName)) + pathInfo = pathInfo[scriptName.length .. $]; + + requestUri = scriptName ~ pathInfo ~ (queryString.length ? ("?" ~ queryString) : ""); + + iis = true; // FIXME HACK - used in byChunk below - see bugzilla 6339 + + // FIXME: this works for apache and iis... but what about others? + } + + + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + getArray = assumeUnique(ugh); + get = keepLastOf(getArray); + + + // NOTE: on shitpache, you need to specifically forward this + authorization = getenv("HTTP_AUTHORIZATION"); + // this is a hack because Apache is a shitload of fuck and + // refuses to send the real header to us. Compatible + // programs should send both the standard and X- versions + + // NOTE: if you have access to .htaccess or httpd.conf, you can make this + // unnecessary with mod_rewrite, so it is commented + + //if(authorization.length == 0) // if the std is there, use it + // authorization = getenv("HTTP_X_AUTHORIZATION"); + + // the REDIRECT_HTTPS check is here because with an Apache hack, the port can become wrong + if(getenv("SERVER_PORT").length && getenv("REDIRECT_HTTPS") != "on") + port = to!int(getenv("SERVER_PORT")); + else + port = 0; // this was probably called from the command line + + auto ae = getenv("HTTP_ACCEPT_ENCODING"); + if(ae.length && ae.indexOf("gzip") != -1) + acceptsGzip = true; + + accept = getenv("HTTP_ACCEPT"); + lastEventId = getenv("HTTP_LAST_EVENT_ID"); + + auto ka = getenv("HTTP_CONNECTION"); + if(ka.length && ka.asLowerCase().canFind("keep-alive")) + keepAliveRequested = true; + + auto or = getenv("HTTP_ORIGIN"); + origin = or; + + auto rm = getenv("REQUEST_METHOD"); + if(rm.length) + requestMethod = to!RequestMethod(getenv("REQUEST_METHOD")); + else + requestMethod = RequestMethod.CommandLine; + + // FIXME: hack on REDIRECT_HTTPS; this is there because the work app uses mod_rewrite which loses the https flag! So I set it with [E=HTTPS=%HTTPS] or whatever but then it gets translated to here so i want it to still work. This is arguably wrong but meh. + https = (getenv("HTTPS") == "on" || getenv("REDIRECT_HTTPS") == "on"); + + // FIXME: DOCUMENT_ROOT? + + // FIXME: what about PUT? + if(requestMethod == RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT || requestMethod == Cgi.RequestMethod.CommandLine) { + version(preserveData) // a hack to make forwarding simpler + immutable(ubyte)[] data; + size_t amountReceived = 0; + auto contentType = getenv("CONTENT_TYPE"); + + // FIXME: is this ever not going to be set? I guess it depends + // on if the server de-chunks and buffers... seems like it has potential + // to be slow if they did that. The spec says it is always there though. + // And it has worked reliably for me all year in the live environment, + // but some servers might be different. + auto cls = getenv("CONTENT_LENGTH"); + auto contentLength = to!size_t(cls.length ? cls : "0"); + + immutable originalContentLength = contentLength; + if(contentLength) { + if(maxContentLength > 0 && contentLength > maxContentLength) { + setResponseStatus("413 Request entity too large"); + write("You tried to upload a file that is too large."); + close(); + throw new Exception("POST too large"); + } + prepareForIncomingDataChunks(contentType, contentLength); + + + int processChunk(in ubyte[] chunk) { + if(chunk.length > contentLength) { + handleIncomingDataChunk(chunk[0..contentLength]); + amountReceived += contentLength; + contentLength = 0; + return 1; + } else { + handleIncomingDataChunk(chunk); + contentLength -= chunk.length; + amountReceived += chunk.length; + } + if(contentLength == 0) + return 1; + + onRequestBodyDataReceived(amountReceived, originalContentLength); + return 0; + } + + + if(readdata is null) { + foreach(ubyte[] chunk; stdin.byChunk(iis ? contentLength : 4096)) + if(processChunk(chunk)) + break; + } else { + // we have a custom data source.. + auto chunk = readdata(); + while(chunk.length) { + if(processChunk(chunk)) + break; + chunk = readdata(); + } + } + + onRequestBodyDataReceived(amountReceived, originalContentLength); + postArray = assumeUnique(pps._post); + filesArray = assumeUnique(pps._files); + files = keepLastOf(filesArray); + post = keepLastOf(postArray); + this.postBody = pps.postBody; + this.requestContentType = contentType; + cleanUpPostDataState(); + } + + version(preserveData) + originalPostData = data; + } + // fixme: remote_user script name + + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + } + + /// Cleans up any temporary files. Do not use the object + /// after calling this. + /// + /// NOTE: it is called automatically by GenericMain + // FIXME: this should be called if the constructor fails too, if it has created some garbage... + void dispose() { + foreach(file; files) { + if(!file.contentInMemory) + if(std.file.exists(file.contentFilename)) + std.file.remove(file.contentFilename); + } + } + + private { + struct PostParserState { + string contentType; + string boundary; + string localBoundary; // the ones used at the end or something lol + bool isMultipart; + bool needsSavedBody; + + ulong expectedLength; + ulong contentConsumed; + immutable(ubyte)[] buffer; + + // multipart parsing state + int whatDoWeWant; + bool weHaveAPart; + string[] thisOnesHeaders; + immutable(ubyte)[] thisOnesData; + + string postBody; + + UploadedFile piece; + bool isFile = false; + + size_t memoryCommitted; + + // do NOT keep mutable references to these anywhere! + // I assume they are unique in the constructor once we're all done getting data. + string[][string] _post; + UploadedFile[][string] _files; + } + + PostParserState pps; + } + + /// This represents a file the user uploaded via a POST request. + static struct UploadedFile { + /// If you want to create one of these structs for yourself from some data, + /// use this function. + static UploadedFile fromData(immutable(void)[] data, string name = null) { + Cgi.UploadedFile f; + f.filename = name; + f.content = cast(immutable(ubyte)[]) data; + f.contentInMemory = true; + return f; + } + + string name; /// The name of the form element. + string filename; /// The filename the user set. + string contentType; /// The MIME type the user's browser reported. (Not reliable.) + + /** + For small files, cgi.d will buffer the uploaded file in memory, and make it + directly accessible to you through the content member. I find this very convenient + and somewhat efficient, since it can avoid hitting the disk entirely. (I + often want to inspect and modify the file anyway!) + + I find the file is very large, it is undesirable to eat that much memory just + for a file buffer. In those cases, if you pass a large enough value for maxContentLength + to the constructor so they are accepted, cgi.d will write the content to a temporary + file that you can re-read later. + + You can override this behavior by subclassing Cgi and overriding the protected + handlePostChunk method. Note that the object is not initialized when you + write that method - the http headers are available, but the cgi.post method + is not. You may parse the file as it streams in using this method. + + + Anyway, if the file is small enough to be in memory, contentInMemory will be + set to true, and the content is available in the content member. + + If not, contentInMemory will be set to false, and the content saved in a file, + whose name will be available in the contentFilename member. + + + Tip: if you know you are always dealing with small files, and want the convenience + of ignoring this member, construct Cgi with a small maxContentLength. Then, if + a large file comes in, it simply throws an exception (and HTTP error response) + instead of trying to handle it. + + The default value of maxContentLength in the constructor is for small files. + */ + bool contentInMemory = true; // the default ought to always be true + immutable(ubyte)[] content; /// The actual content of the file, if contentInMemory == true + string contentFilename; /// the file where we dumped the content, if contentInMemory == false. Note that if you want to keep it, you MUST move the file, since otherwise it is considered garbage when cgi is disposed. + + /// + ulong fileSize() const { + if(contentInMemory) + return content.length; + import std.file; + return std.file.getSize(contentFilename); + + } + + /// + void writeToFile(string filenameToSaveTo) const { + import std.file; + if(contentInMemory) + std.file.write(filenameToSaveTo, content); + else + std.file.rename(contentFilename, filenameToSaveTo); + } + } + + // given a content type and length, decide what we're going to do with the data.. + protected void prepareForIncomingDataChunks(string contentType, ulong contentLength) { + pps.expectedLength = contentLength; + + auto terminator = contentType.indexOf(";"); + if(terminator == -1) + terminator = contentType.length; + + pps.contentType = contentType[0 .. terminator]; + auto b = contentType[terminator .. $]; + if(b.length) { + auto idx = b.indexOf("boundary="); + if(idx != -1) { + pps.boundary = b[idx + "boundary=".length .. $]; + pps.localBoundary = "\r\n--" ~ pps.boundary; + } + } + + // while a content type SHOULD be sent according to the RFC, it is + // not required. We're told we SHOULD guess by looking at the content + // but it seems to me that this only happens when it is urlencoded. + if(pps.contentType == "application/x-www-form-urlencoded" || pps.contentType == "") { + pps.isMultipart = false; + pps.needsSavedBody = false; + } else if(pps.contentType == "multipart/form-data") { + pps.isMultipart = true; + enforce(pps.boundary.length, "no boundary"); + } else if(pps.contentType == "text/xml") { // FIXME: could this be special and load the post params + // save the body so the application can handle it + pps.isMultipart = false; + pps.needsSavedBody = true; + } else if(pps.contentType == "application/json") { // FIXME: this could prolly try to load post params too + // save the body so the application can handle it + pps.needsSavedBody = true; + pps.isMultipart = false; + } else { + // the rest is 100% handled by the application. just save the body and send it to them + pps.needsSavedBody = true; + pps.isMultipart = false; + } + } + + // handles streaming POST data. If you handle some other content type, you should + // override this. If the data isn't the content type you want, you ought to call + // super.handleIncomingDataChunk so regular forms and files still work. + + // FIXME: I do some copying in here that I'm pretty sure is unnecessary, and the + // file stuff I'm sure is inefficient. But, my guess is the real bottleneck is network + // input anyway, so I'm not going to get too worked up about it right now. + protected void handleIncomingDataChunk(const(ubyte)[] chunk) { + if(chunk.length == 0) + return; + assert(chunk.length <= 32 * 1024 * 1024); // we use chunk size as a memory constraint thing, so + // if we're passed big chunks, it might throw unnecessarily. + // just pass it smaller chunks at a time. + if(pps.isMultipart) { + // multipart/form-data + + + // FIXME: this might want to be factored out and factorized + // need to make sure the stream hooks actually work. + void pieceHasNewContent() { + // we just grew the piece's buffer. Do we have to switch to file backing? + if(pps.piece.contentInMemory) { + if(pps.piece.content.length <= 10 * 1024 * 1024) + // meh, I'm ok with it. + return; + else { + // this is too big. + if(!pps.isFile) + throw new Exception("Request entity too large"); // a variable this big is kinda ridiculous, just reject it. + else { + // a file this large is probably acceptable though... let's use a backing file. + pps.piece.contentInMemory = false; + // FIXME: say... how do we intend to delete these things? cgi.dispose perhaps. + + int count = 0; + pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count); + // odds are this loop will never be entered, but we want it just in case. + while(std.file.exists(pps.piece.contentFilename)) { + count++; + pps.piece.contentFilename = getTempDirectory() ~ "arsd_cgi_uploaded_file_" ~ to!string(getUtcTime()) ~ "-" ~ to!string(count); + } + // I hope this creates the file pretty quickly, or the loop might be useless... + // FIXME: maybe I should write some kind of custom transaction here. + std.file.write(pps.piece.contentFilename, pps.piece.content); + + pps.piece.content = null; + } + } + } else { + // it's already in a file, so just append it to what we have + if(pps.piece.content.length) { + // FIXME: this is surely very inefficient... we'll be calling this by 4kb chunk... + std.file.append(pps.piece.contentFilename, pps.piece.content); + pps.piece.content = null; + } + } + } + + + void commitPart() { + if(!pps.weHaveAPart) + return; + + pieceHasNewContent(); // be sure the new content is handled every time + + if(pps.isFile) { + // I'm not sure if other environments put files in post or not... + // I used to not do it, but I think I should, since it is there... + pps._post[pps.piece.name] ~= pps.piece.filename; + pps._files[pps.piece.name] ~= pps.piece; + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= pps.piece.filename; + } else { + pps._post[pps.piece.name] ~= cast(string) pps.piece.content; + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= cast(string) pps.piece.content; + } + + /* + stderr.writeln("RECEIVED: ", pps.piece.name, "=", + pps.piece.content.length < 1000 + ? + to!string(pps.piece.content) + : + "too long"); + */ + + // FIXME: the limit here + pps.memoryCommitted += pps.piece.content.length; + + pps.weHaveAPart = false; + pps.whatDoWeWant = 1; + pps.thisOnesHeaders = null; + pps.thisOnesData = null; + + pps.piece = UploadedFile.init; + pps.isFile = false; + } + + void acceptChunk() { + pps.buffer ~= chunk; + chunk = null; // we've consumed it into the buffer, so keeping it just brings confusion + } + + immutable(ubyte)[] consume(size_t howMuch) { + pps.contentConsumed += howMuch; + auto ret = pps.buffer[0 .. howMuch]; + pps.buffer = pps.buffer[howMuch .. $]; + return ret; + } + + dataConsumptionLoop: do { + switch(pps.whatDoWeWant) { + default: assert(0); + case 0: + acceptChunk(); + // the format begins with two extra leading dashes, then we should be at the boundary + if(pps.buffer.length < 2) + return; + assert(pps.buffer[0] == '-', "no leading dash"); + consume(1); + assert(pps.buffer[0] == '-', "no second leading dash"); + consume(1); + + pps.whatDoWeWant = 1; + goto case 1; + /* fallthrough */ + case 1: // looking for headers + // here, we should be lined up right at the boundary, which is followed by a \r\n + + // want to keep the buffer under control in case we're under attack + //stderr.writeln("here once"); + //if(pps.buffer.length + chunk.length > 70 * 1024) // they should be < 1 kb really.... + // throw new Exception("wtf is up with the huge mime part headers"); + + acceptChunk(); + + if(pps.buffer.length < pps.boundary.length) + return; // not enough data, since there should always be a boundary here at least + + if(pps.contentConsumed + pps.boundary.length + 6 == pps.expectedLength) { + assert(pps.buffer.length == pps.boundary.length + 4 + 2); // --, --, and \r\n + // we *should* be at the end here! + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + + // the message is terminated by --BOUNDARY--\r\n (after a \r\n leading to the boundary) + assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary, + "not lined up on boundary " ~ pps.boundary); + consume(pps.boundary.length); + + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + + assert(pps.buffer.length == 0); + assert(pps.contentConsumed == pps.expectedLength); + break dataConsumptionLoop; // we're done! + } else { + // we're not done yet. We should be lined up on a boundary. + + // But, we want to ensure the headers are here before we consume anything! + auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n"); + if(headerEndLocation == -1) + return; // they *should* all be here, so we can handle them all at once. + + assert(pps.buffer[0 .. pps.boundary.length] == cast(const(ubyte[])) pps.boundary, + "not lined up on boundary " ~ pps.boundary); + + consume(pps.boundary.length); + // the boundary is always followed by a \r\n + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + } + + // re-running since by consuming the boundary, we invalidate the old index. + auto headerEndLocation = locationOf(pps.buffer, "\r\n\r\n"); + assert(headerEndLocation >= 0, "no header"); + auto thisOnesHeaders = pps.buffer[0..headerEndLocation]; + + consume(headerEndLocation + 4); // The +4 is the \r\n\r\n that caps it off + + pps.thisOnesHeaders = split(cast(string) thisOnesHeaders, "\r\n"); + + // now we'll parse the headers + foreach(h; pps.thisOnesHeaders) { + auto p = h.indexOf(":"); + assert(p != -1, "no colon in header, got " ~ to!string(pps.thisOnesHeaders)); + string hn = h[0..p]; + string hv = h[p+2..$]; + + switch(hn.toLower) { + default: assert(0); + case "content-disposition": + auto info = hv.split("; "); + foreach(i; info[1..$]) { // skipping the form-data + auto o = i.split("="); // FIXME + string pn = o[0]; + string pv = o[1][1..$-1]; + + if(pn == "name") { + pps.piece.name = pv; + } else if (pn == "filename") { + pps.piece.filename = pv; + pps.isFile = true; + } + } + break; + case "content-type": + pps.piece.contentType = hv; + break; + } + } + + pps.whatDoWeWant++; // move to the next step - the data + break; + case 2: + // when we get here, pps.buffer should contain our first chunk of data + + if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // we might buffer quite a bit but not much + throw new Exception("wtf is up with the huge mime part buffer"); + + acceptChunk(); + + // so the trick is, we want to process all the data up to the boundary, + // but what if the chunk's end cuts the boundary off? If we're unsure, we + // want to wait for the next chunk. We start by looking for the whole boundary + // in the buffer somewhere. + + auto boundaryLocation = locationOf(pps.buffer, pps.localBoundary); + // assert(boundaryLocation != -1, "should have seen "~to!string(cast(ubyte[]) pps.localBoundary)~" in " ~ to!string(pps.buffer)); + if(boundaryLocation != -1) { + // this is easy - we can see it in it's entirety! + + pps.piece.content ~= consume(boundaryLocation); + + assert(pps.buffer[0] == '\r'); + consume(1); + assert(pps.buffer[0] == '\n'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + assert(pps.buffer[0] == '-'); + consume(1); + // the boundary here is always preceded by \r\n--, which is why we used localBoundary instead of boundary to locate it. Cut that off. + pps.weHaveAPart = true; + pps.whatDoWeWant = 1; // back to getting headers for the next part + + commitPart(); // we're done here + } else { + // we can't see the whole thing, but what if there's a partial boundary? + + enforce(pps.localBoundary.length < 128); // the boundary ought to be less than a line... + assert(pps.localBoundary.length > 1); // should already be sane but just in case + bool potentialBoundaryFound = false; + + boundaryCheck: for(int a = 1; a < pps.localBoundary.length; a++) { + // we grow the boundary a bit each time. If we think it looks the + // same, better pull another chunk to be sure it's not the end. + // Starting small because exiting the loop early is desirable, since + // we're not keeping any ambiguity and 1 / 256 chance of exiting is + // the best we can do. + if(a > pps.buffer.length) + break; // FIXME: is this right? + assert(a <= pps.buffer.length); + assert(a > 0); + if(std.algorithm.endsWith(pps.buffer, pps.localBoundary[0 .. a])) { + // ok, there *might* be a boundary here, so let's + // not treat the end as data yet. The rest is good to + // use though, since if there was a boundary there, we'd + // have handled it up above after locationOf. + + pps.piece.content ~= pps.buffer[0 .. $ - a]; + consume(pps.buffer.length - a); + pieceHasNewContent(); + potentialBoundaryFound = true; + break boundaryCheck; + } + } + + if(!potentialBoundaryFound) { + // we can consume the whole thing + pps.piece.content ~= pps.buffer; + pieceHasNewContent(); + consume(pps.buffer.length); + } else { + // we found a possible boundary, but there was + // insufficient data to be sure. + assert(pps.buffer == cast(const(ubyte[])) pps.localBoundary[0 .. pps.buffer.length]); + + return; // wait for the next chunk. + } + } + } + } while(pps.buffer.length); + + // btw all boundaries except the first should have a \r\n before them + } else { + // application/x-www-form-urlencoded and application/json + + // not using maxContentLength because that might be cranked up to allow + // large file uploads. We can handle them, but a huge post[] isn't any good. + if(pps.buffer.length + chunk.length > 8 * 1024 * 1024) // surely this is plenty big enough + throw new Exception("wtf is up with such a gigantic form submission????"); + + pps.buffer ~= chunk; + + // simple handling, but it works... until someone bombs us with gigabytes of crap at least... + if(pps.buffer.length == pps.expectedLength) { + if(pps.needsSavedBody) + pps.postBody = cast(string) pps.buffer; + else + pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder); + version(preserveData) + originalPostData = pps.buffer; + } else { + // just for debugging + } + } + } + + protected void cleanUpPostDataState() { + pps = PostParserState.init; + } + + /// you can override this function to somehow react + /// to an upload in progress. + /// + /// Take note that parts of the CGI object is not yet + /// initialized! Stuff from HTTP headers, including get[], is usable. + /// But, none of post[] is usable, and you cannot write here. That's + /// why this method is const - mutating the object won't do much anyway. + /// + /// My idea here was so you can output a progress bar or + /// something to a cooperative client (see arsd.rtud for a potential helper) + /// + /// The default is to do nothing. Subclass cgi and use the + /// CustomCgiMain mixin to do something here. + void onRequestBodyDataReceived(size_t receivedSoFar, size_t totalExpected) const { + // This space intentionally left blank. + } + + /// Initializes the cgi from completely raw HTTP data. The ir must have a Socket source. + /// *closeConnection will be set to true if you should close the connection after handling this request + this(BufferedInputRange ir, bool* closeConnection) { + isCalledWithCommandLineArguments = false; + import al = std.algorithm; + + immutable(ubyte)[] data; + + void rdo(const(ubyte)[] d) { + //import std.stdio; writeln(d); + sendAll(ir.source, d); + } + + auto ira = ir.source.remoteAddress(); + auto irLocalAddress = ir.source.localAddress(); + + ushort port = 80; + if(auto ia = cast(InternetAddress) irLocalAddress) { + port = ia.port; + } else if(auto ia = cast(Internet6Address) irLocalAddress) { + port = ia.port; + } + + // that check for UnixAddress is to work around a Phobos bug + // see: https://github.com/dlang/phobos/pull/7383 + // but this might be more useful anyway tbh for this case + version(Posix) + this(ir, ira is null ? null : cast(UnixAddress) ira ? "unix:" : ira.toString(), port, 0, false, &rdo, null, closeConnection); + else + this(ir, ira is null ? null : ira.toString(), port, 0, false, &rdo, null, closeConnection); + } + + /** + Initializes it from raw HTTP request data. GenericMain uses this when you compile with -version=embedded_httpd. + + NOTE: If you are behind a reverse proxy, the values here might not be what you expect.... it will use X-Forwarded-For for remote IP and X-Forwarded-Host for host + + Params: + inputData = the incoming data, including headers and other raw http data. + When the constructor exits, it will leave this range exactly at the start of + the next request on the connection (if there is one). + + address = the IP address of the remote user + _port = the port number of the connection + pathInfoStarts = the offset into the path component of the http header where the SCRIPT_NAME ends and the PATH_INFO begins. + _https = if this connection is encrypted (note that the input data must not actually be encrypted) + _rawDataOutput = delegate to accept response data. It should write to the socket or whatever; Cgi does all the needed processing to speak http. + _flush = if _rawDataOutput buffers, this delegate should flush the buffer down the wire + closeConnection = if the request asks to close the connection, *closeConnection == true. + */ + this( + BufferedInputRange inputData, +// string[] headers, immutable(ubyte)[] data, + string address, ushort _port, + int pathInfoStarts = 0, // use this if you know the script name, like if this is in a folder in a bigger web environment + bool _https = false, + void delegate(const(ubyte)[]) _rawDataOutput = null, + void delegate() _flush = null, + // this pointer tells if the connection is supposed to be closed after we handle this + bool* closeConnection = null) + { + // these are all set locally so the loop works + // without triggering errors in dmd 2.064 + // we go ahead and set them at the end of it to the this version + int port; + string referrer; + string remoteAddress; + string userAgent; + string authorization; + string origin; + string accept; + string lastEventId; + bool https; + string host; + RequestMethod requestMethod; + string requestUri; + string pathInfo; + string queryString; + string scriptName; + string[string] get; + string[][string] getArray; + bool keepAliveRequested; + bool acceptsGzip; + string cookie; + + + + environmentVariables = cast(const) environment.toAA; + + idlol = inputData; + + isCalledWithCommandLineArguments = false; + + https = _https; + port = _port; + + rawDataOutput = _rawDataOutput; + flushDelegate = _flush; + nph = true; + + remoteAddress = address; + + // streaming parser + import al = std.algorithm; + + // FIXME: tis cast is technically wrong, but Phobos deprecated al.indexOf... for some reason. + auto idx = indexOf(cast(string) inputData.front(), "\r\n\r\n"); + while(idx == -1) { + inputData.popFront(0); + idx = indexOf(cast(string) inputData.front(), "\r\n\r\n"); + } + + assert(idx != -1); + + + string contentType = ""; + string[string] requestHeadersHere; + + size_t contentLength; + + bool isChunked; + + { + import core.runtime; + scriptFileName = Runtime.args.length ? Runtime.args[0] : null; + } + + + int headerNumber = 0; + foreach(line; al.splitter(inputData.front()[0 .. idx], "\r\n")) + if(line.length) { + headerNumber++; + auto header = cast(string) line.idup; + if(headerNumber == 1) { + // request line + auto parts = al.splitter(header, " "); + if(parts.front == "PRI") { + // this is an HTTP/2.0 line - "PRI * HTTP/2.0" - which indicates their payload will follow + // we're going to immediately refuse this, im not interested in implementing http2 (it is unlikely + // to bring me benefit) + throw new HttpVersionNotSupportedException(); + } + requestMethod = to!RequestMethod(parts.front); + parts.popFront(); + requestUri = parts.front; + + // FIXME: the requestUri could be an absolute path!!! should I rename it or something? + scriptName = requestUri[0 .. pathInfoStarts]; + + auto question = requestUri.indexOf("?"); + if(question == -1) { + queryString = ""; + // FIXME: double check, this might be wrong since it could be url encoded + pathInfo = requestUri[pathInfoStarts..$]; + } else { + queryString = requestUri[question+1..$]; + pathInfo = requestUri[pathInfoStarts..question]; + } + + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); + getArray = cast(string[][string]) assumeUnique(ugh); + + if(header.indexOf("HTTP/1.0") != -1) { + http10 = true; + autoBuffer = true; + if(closeConnection) { + // on http 1.0, close is assumed (unlike http/1.1 where we assume keep alive) + *closeConnection = true; + } + } + } else { + // other header + auto colon = header.indexOf(":"); + if(colon == -1) + throw new Exception("HTTP headers should have a colon!"); + string name = header[0..colon].toLower; + string value = header[colon+2..$]; // skip the colon and the space + + requestHeadersHere[name] = value; + + if (name == "accept") { + accept = value; + } + else if (name == "origin") { + origin = value; + } + else if (name == "connection") { + if(value == "close" && closeConnection) + *closeConnection = true; + if(value.asLowerCase().canFind("keep-alive")) { + keepAliveRequested = true; + + // on http 1.0, the connection is closed by default, + // but not if they request keep-alive. then we don't close + // anymore - undoing the set above + if(http10 && closeConnection) { + *closeConnection = false; + } + } + } + else if (name == "transfer-encoding") { + if(value == "chunked") + isChunked = true; + } + else if (name == "last-event-id") { + lastEventId = value; + } + else if (name == "authorization") { + authorization = value; + } + else if (name == "content-type") { + contentType = value; + } + else if (name == "content-length") { + contentLength = to!size_t(value); + } + else if (name == "x-forwarded-for") { + remoteAddress = value; + } + else if (name == "x-forwarded-host" || name == "host") { + if(name != "host" || host is null) + host = value; + } + // FIXME: https://tools.ietf.org/html/rfc7239 + else if (name == "accept-encoding") { + if(value.indexOf("gzip") != -1) + acceptsGzip = true; + } + else if (name == "user-agent") { + userAgent = value; + } + else if (name == "referer") { + referrer = value; + } + else if (name == "cookie") { + cookie ~= value; + } else if(name == "expect") { + if(value == "100-continue") { + // FIXME we should probably give user code a chance + // to process and reject but that needs to be virtual, + // perhaps part of the CGI redesign. + + // FIXME: if size is > max content length it should + // also fail at this point. + _rawDataOutput(cast(ubyte[]) "HTTP/1.1 100 Continue\r\n\r\n"); + + // FIXME: let the user write out 103 early hints too + } + } + // else + // ignore it + + } + } + + inputData.consume(idx + 4); + // done + + requestHeaders = assumeUnique(requestHeadersHere); + + ByChunkRange dataByChunk; + + // reading Content-Length type data + // We need to read up the data we have, and write it out as a chunk. + if(!isChunked) { + dataByChunk = byChunk(inputData, contentLength); + } else { + // chunked requests happen, but not every day. Since we need to know + // the content length (for now, maybe that should change), we'll buffer + // the whole thing here instead of parse streaming. (I think this is what Apache does anyway in cgi modes) + auto data = dechunk(inputData); + + // set the range here + dataByChunk = byChunk(data); + contentLength = data.length; + } + + assert(dataByChunk !is null); + + if(contentLength) { + prepareForIncomingDataChunks(contentType, contentLength); + foreach(dataChunk; dataByChunk) { + handleIncomingDataChunk(dataChunk); + } + postArray = assumeUnique(pps._post); + filesArray = assumeUnique(pps._files); + files = keepLastOf(filesArray); + post = keepLastOf(postArray); + postBody = pps.postBody; + this.requestContentType = contentType; + + cleanUpPostDataState(); + } + + this.port = port; + this.referrer = referrer; + this.remoteAddress = remoteAddress; + this.userAgent = userAgent; + this.authorization = authorization; + this.origin = origin; + this.accept = accept; + this.lastEventId = lastEventId; + this.https = https; + this.host = host; + this.requestMethod = requestMethod; + this.requestUri = requestUri; + this.pathInfo = pathInfo; + this.queryString = queryString; + + this.scriptName = scriptName; + this.get = keepLastOf(getArray); + this.getArray = cast(immutable) getArray; + this.keepAliveRequested = keepAliveRequested; + this.acceptsGzip = acceptsGzip; + this.cookie = cookie; + + cookiesArray = getCookieArray(); + cookies = keepLastOf(cookiesArray); + + } + BufferedInputRange idlol; + + private immutable(string[string]) keepLastOf(in string[][string] arr) { + string[string] ca; + foreach(k, v; arr) + ca[k] = v[$-1]; + + return assumeUnique(ca); + } + + // FIXME duplication + private immutable(UploadedFile[string]) keepLastOf(in UploadedFile[][string] arr) { + UploadedFile[string] ca; + foreach(k, v; arr) + ca[k] = v[$-1]; + + return assumeUnique(ca); + } + + + private immutable(string[][string]) getCookieArray() { + auto forTheLoveOfGod = decodeVariables(cookie, "; "); + return assumeUnique(forTheLoveOfGod); + } + + /++ + Very simple method to require a basic auth username and password. + If the http request doesn't include the required credentials, it throws a + HTTP 401 error, and an exception to cancel your handler. Do NOT catch the + `AuthorizationRequiredException` exception thrown by this if you want the + http basic auth prompt to work for the user! + + Note: basic auth does not provide great security, especially over unencrypted HTTP; + the user's credentials are sent in plain text on every request. + + If you are using Apache, the HTTP_AUTHORIZATION variable may not be sent to the + application. Either use Apache's built in methods for basic authentication, or add + something along these lines to your server configuration: + + ``` + RewriteEngine On + RewriteCond %{HTTP:Authorization} ^(.*) + RewriteRule ^(.*) - [E=HTTP_AUTHORIZATION:%1] + ``` + + To ensure the necessary data is available to cgi.d. + +/ + void requireBasicAuth(string user, string pass, string message = null, string file = __FILE__, size_t line = __LINE__) { + if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) { + throw new AuthorizationRequiredException("Basic", message, file, line); + } + } + + /// Very simple caching controls - setCache(false) means it will never be cached. Good for rapidly updated or sensitive sites. + /// setCache(true) means it will always be cached for as long as possible. Best for static content. + /// Use setResponseExpires and updateResponseExpires for more control + void setCache(bool allowCaching) { + noCache = !allowCaching; + } + + /// Set to true and use cgi.write(data, true); to send a gzipped response to browsers + /// who can accept it + bool gzipResponse; + + immutable bool acceptsGzip; + immutable bool keepAliveRequested; + + /// Set to true if and only if this was initialized with command line arguments + immutable bool isCalledWithCommandLineArguments; + + /// This gets a full url for the current request, including port, protocol, host, path, and query + string getCurrentCompleteUri() const { + ushort defaultPort = https ? 443 : 80; + + string uri = "http"; + if(https) + uri ~= "s"; + uri ~= "://"; + uri ~= host; + /+ // the host has the port so p sure this never needed, cgi on apache and embedded http all do the right hting now + version(none) + if(!(!port || port == defaultPort)) { + uri ~= ":"; + uri ~= to!string(port); + } + +/ + uri ~= requestUri; + return uri; + } + + /// You can override this if your site base url isn't the same as the script name + string logicalScriptName() const { + return scriptName; + } + + /++ + Sets the HTTP status of the response. For example, "404 File Not Found" or "500 Internal Server Error". + It assumes "200 OK", and automatically changes to "302 Found" if you call setResponseLocation(). + Note setResponseStatus() must be called *before* you write() any data to the output. + + History: + The `int` overload was added on January 11, 2021. + +/ + void setResponseStatus(string status) { + assert(!outputtedResponseData); + responseStatus = status; + } + /// ditto + void setResponseStatus(int statusCode) { + setResponseStatus(getHttpCodeText(statusCode)); + } + private string responseStatus = null; + + /// Returns true if it is still possible to output headers + bool canOutputHeaders() { + return !isClosed && !outputtedResponseData; + } + + /// Sets the location header, which the browser will redirect the user to automatically. + /// Note setResponseLocation() must be called *before* you write() any data to the output. + /// The optional important argument is used if it's a default suggestion rather than something to insist upon. + void setResponseLocation(string uri, bool important = true, string status = null) { + if(!important && isCurrentResponseLocationImportant) + return; // important redirects always override unimportant ones + + if(uri is null) { + responseStatus = "200 OK"; + responseLocation = null; + isCurrentResponseLocationImportant = important; + return; // this just cancels the redirect + } + + assert(!outputtedResponseData); + if(status is null) + responseStatus = "302 Found"; + else + responseStatus = status; + + responseLocation = uri.strip; + isCurrentResponseLocationImportant = important; + } + protected string responseLocation = null; + private bool isCurrentResponseLocationImportant = false; + + /// Sets the Expires: http header. See also: updateResponseExpires, setPublicCaching + /// The parameter is in unix_timestamp * 1000. Try setResponseExpires(getUTCtime() + SOME AMOUNT) for normal use. + /// Note: the when parameter is different than setCookie's expire parameter. + void setResponseExpires(long when, bool isPublic = false) { + responseExpires = when; + setCache(true); // need to enable caching so the date has meaning + + responseIsPublic = isPublic; + responseExpiresRelative = false; + } + + /// Sets a cache-control max-age header for whenFromNow, in seconds. + void setResponseExpiresRelative(int whenFromNow, bool isPublic = false) { + responseExpires = whenFromNow; + setCache(true); // need to enable caching so the date has meaning + + responseIsPublic = isPublic; + responseExpiresRelative = true; + } + private long responseExpires = long.min; + private bool responseIsPublic = false; + private bool responseExpiresRelative = false; + + /// This is like setResponseExpires, but it can be called multiple times. The setting most in the past is the one kept. + /// If you have multiple functions, they all might call updateResponseExpires about their own return value. The program + /// output as a whole is as cacheable as the least cachable part in the chain. + + /// setCache(false) always overrides this - it is, by definition, the strictest anti-cache statement available. If your site outputs sensitive user data, you should probably call setCache(false) when you do, to ensure no other functions will cache the content, as it may be a privacy risk. + /// Conversely, setting here overrides setCache(true), since any expiration date is in the past of infinity. + void updateResponseExpires(long when, bool isPublic) { + if(responseExpires == long.min) + setResponseExpires(when, isPublic); + else if(when < responseExpires) + setResponseExpires(when, responseIsPublic && isPublic); // if any part of it is private, it all is + } + + /* + /// Set to true if you want the result to be cached publically - that is, is the content shared? + /// Should generally be false if the user is logged in. It assumes private cache only. + /// setCache(true) also turns on public caching, and setCache(false) sets to private. + void setPublicCaching(bool allowPublicCaches) { + publicCaching = allowPublicCaches; + } + private bool publicCaching = false; + */ + + /++ + History: + Added January 11, 2021 + +/ + enum SameSitePolicy { + Lax, + Strict, + None + } + + /++ + Sets an HTTP cookie, automatically encoding the data to the correct string. + expiresIn is how many milliseconds in the future the cookie will expire. + TIP: to make a cookie accessible from subdomains, set the domain to .yourdomain.com. + Note setCookie() must be called *before* you write() any data to the output. + + History: + Parameter `sameSitePolicy` was added on January 11, 2021. + +/ + void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false, SameSitePolicy sameSitePolicy = SameSitePolicy.Lax) { + assert(!outputtedResponseData); + string cookie = encodeUriComponent(name) ~ "="; + cookie ~= encodeUriComponent(data); + if(path !is null) + cookie ~= "; path=" ~ path; + // FIXME: should I just be using max-age here? (also in cache below) + if(expiresIn != 0) + cookie ~= "; expires=" ~ printDate(cast(DateTime) Clock.currTime(UTC()) + dur!"msecs"(expiresIn)); + if(domain !is null) + cookie ~= "; domain=" ~ domain; + if(secure == true) + cookie ~= "; Secure"; + if(httpOnly == true ) + cookie ~= "; HttpOnly"; + final switch(sameSitePolicy) { + case SameSitePolicy.Lax: + cookie ~= "; SameSite=Lax"; + break; + case SameSitePolicy.Strict: + cookie ~= "; SameSite=Strict"; + break; + case SameSitePolicy.None: + cookie ~= "; SameSite=None"; + assert(secure); // cookie spec requires this now, see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite + break; + } + + if(auto idx = name in cookieIndexes) { + responseCookies[*idx] = cookie; + } else { + cookieIndexes[name] = responseCookies.length; + responseCookies ~= cookie; + } + } + private string[] responseCookies; + private size_t[string] cookieIndexes; + + /// Clears a previously set cookie with the given name, path, and domain. + void clearCookie(string name, string path = null, string domain = null) { + assert(!outputtedResponseData); + setCookie(name, "", 1, path, domain); + } + + /// Sets the content type of the response, for example "text/html" (the default) for HTML, or "image/png" for a PNG image + void setResponseContentType(string ct) { + assert(!outputtedResponseData); + responseContentType = ct; + } + private string responseContentType = null; + + /// Adds a custom header. It should be the name: value, but without any line terminator. + /// For example: header("X-My-Header: Some value"); + /// Note you should use the specialized functions in this object if possible to avoid + /// duplicates in the output. + void header(string h) { + customHeaders ~= h; + } + + /++ + I named the original function `header` after PHP, but this pattern more fits + the rest of the Cgi object. + + Either name are allowed. + + History: + Alias added June 17, 2022. + +/ + alias setResponseHeader = header; + + private string[] customHeaders; + private bool websocketMode; + + void flushHeaders(const(void)[] t, bool isAll = false) { + StackBuffer buffer = StackBuffer(0); + + prepHeaders(t, isAll, &buffer); + + if(rawDataOutput !is null) + rawDataOutput(cast(const(ubyte)[]) buffer.get()); + else { + stdout.rawWrite(buffer.get()); + } + } + + private void prepHeaders(const(void)[] t, bool isAll, StackBuffer* buffer) { + string terminator = "\n"; + if(rawDataOutput !is null) + terminator = "\r\n"; + + if(responseStatus !is null) { + if(nph) { + if(http10) + buffer.add("HTTP/1.0 ", responseStatus, terminator); + else + buffer.add("HTTP/1.1 ", responseStatus, terminator); + } else + buffer.add("Status: ", responseStatus, terminator); + } else if (nph) { + if(http10) + buffer.add("HTTP/1.0 200 OK", terminator); + else + buffer.add("HTTP/1.1 200 OK", terminator); + } + + if(websocketMode) + goto websocket; + + if(nph) { // we're responsible for setting the date too according to http 1.1 + char[29] db = void; + printDateToBuffer(cast(DateTime) Clock.currTime(UTC()), db[]); + buffer.add("Date: ", db[], terminator); + } + + // FIXME: what if the user wants to set his own content-length? + // The custom header function can do it, so maybe that's best. + // Or we could reuse the isAll param. + if(responseLocation !is null) { + buffer.add("Location: ", responseLocation, terminator); + } + if(!noCache && responseExpires != long.min) { // an explicit expiration date is set + if(responseExpiresRelative) { + buffer.add("Cache-Control: ", responseIsPublic ? "public" : "private", ", max-age="); + buffer.add(responseExpires); + buffer.add(", no-cache=\"set-cookie, set-cookie2\"", terminator); + } else { + auto expires = SysTime(unixTimeToStdTime(cast(int)(responseExpires / 1000)), UTC()); + char[29] db = void; + printDateToBuffer(cast(DateTime) expires, db[]); + buffer.add("Expires: ", db[], terminator); + // FIXME: assuming everything is private unless you use nocache - generally right for dynamic pages, but not necessarily + buffer.add("Cache-Control: ", (responseIsPublic ? "public" : "private"), ", no-cache=\"set-cookie, set-cookie2\""); + buffer.add(terminator); + } + } + if(responseCookies !is null && responseCookies.length > 0) { + foreach(c; responseCookies) + buffer.add("Set-Cookie: ", c, terminator); + } + if(noCache) { // we specifically do not want caching (this is actually the default) + buffer.add("Cache-Control: private, no-cache=\"set-cookie\"", terminator); + buffer.add("Expires: 0", terminator); + buffer.add("Pragma: no-cache", terminator); + } else { + if(responseExpires == long.min) { // caching was enabled, but without a date set - that means assume cache forever + buffer.add("Cache-Control: public", terminator); + buffer.add("Expires: Tue, 31 Dec 2030 14:00:00 GMT", terminator); // FIXME: should not be more than one year in the future + } + } + if(responseContentType !is null) { + buffer.add("Content-Type: ", responseContentType, terminator); + } else + buffer.add("Content-Type: text/html; charset=utf-8", terminator); + + if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary + buffer.add("Content-Encoding: gzip", terminator); + } + + + if(!isAll) { + if(nph && !http10) { + buffer.add("Transfer-Encoding: chunked", terminator); + responseChunked = true; + } + } else { + buffer.add("Content-Length: "); + buffer.add(t.length); + buffer.add(terminator); + if(nph && keepAliveRequested) { + buffer.add("Connection: Keep-Alive", terminator); + } + } + + websocket: + + foreach(hd; customHeaders) + buffer.add(hd, terminator); + + // FIXME: what about duplicated headers? + + // end of header indicator + buffer.add(terminator); + + outputtedResponseData = true; + } + + /// Writes the data to the output, flushing headers if they have not yet been sent. + void write(const(void)[] t, bool isAll = false, bool maybeAutoClose = true) { + assert(!closed, "Output has already been closed"); + + StackBuffer buffer = StackBuffer(0); + + if(gzipResponse && acceptsGzip && isAll) { // FIXME: isAll really shouldn't be necessary + // actually gzip the data here + + auto c = new Compress(HeaderFormat.gzip); // want gzip + + auto data = c.compress(t); + data ~= c.flush(); + + // std.file.write("/tmp/last-item", data); + + t = data; + } + + if(!outputtedResponseData && (!autoBuffer || isAll)) { + prepHeaders(t, isAll, &buffer); + } + + if(requestMethod != RequestMethod.HEAD && t.length > 0) { + if (autoBuffer && !isAll) { + outputBuffer ~= cast(ubyte[]) t; + } + if(!autoBuffer || isAll) { + if(rawDataOutput !is null) + if(nph && responseChunked) { + //rawDataOutput(makeChunk(cast(const(ubyte)[]) t)); + // we're making the chunk here instead of in a function + // to avoid unneeded gc pressure + buffer.add(toHex(t.length)); + buffer.add("\r\n"); + buffer.add(cast(char[]) t, "\r\n"); + } else { + buffer.add(cast(char[]) t); + } + else + buffer.add(cast(char[]) t); + } + } + + if(rawDataOutput !is null) + rawDataOutput(cast(const(ubyte)[]) buffer.get()); + else + stdout.rawWrite(buffer.get()); + + if(maybeAutoClose && isAll) + close(); // if you say it is all, that means we're definitely done + // maybeAutoClose can be false though to avoid this (important if you call from inside close()! + } + + /++ + Convenience method to set content type to json and write the string as the complete response. + + History: + Added January 16, 2020 + +/ + void writeJson(string json) { + this.setResponseContentType("application/json"); + this.write(json, true); + } + + /// Flushes the pending buffer, leaving the connection open so you can send more. + void flush() { + if(rawDataOutput is null) + stdout.flush(); + else if(flushDelegate !is null) + flushDelegate(); + } + + version(autoBuffer) + bool autoBuffer = true; + else + bool autoBuffer = false; + ubyte[] outputBuffer; + + /// Flushes the buffers to the network, signifying that you are done. + /// You should always call this explicitly when you are done outputting data. + void close() { + if(closed) + return; // don't double close + + if(!outputtedResponseData) + write("", true, false); + + // writing auto buffered data + if(requestMethod != RequestMethod.HEAD && autoBuffer) { + if(!nph) + stdout.rawWrite(outputBuffer); + else + write(outputBuffer, true, false); // tell it this is everything + } + + // closing the last chunk... + if(nph && rawDataOutput !is null && responseChunked) + rawDataOutput(cast(const(ubyte)[]) "0\r\n\r\n"); + + if(flushDelegate) + flushDelegate(); + + closed = true; + } + + // Closes without doing anything, shouldn't be used often + void rawClose() { + closed = true; + } + + /++ + Gets a request variable as a specific type, or the default value of it isn't there + or isn't convertible to the request type. + + Checks both GET and POST variables, preferring the POST variable, if available. + + A nice trick is using the default value to choose the type: + + --- + /* + The return value will match the type of the default. + Here, I gave 10 as a default, so the return value will + be an int. + + If the user-supplied value cannot be converted to the + requested type, you will get the default value back. + */ + int a = cgi.request("number", 10); + + if(cgi.get["number"] == "11") + assert(a == 11); // conversion succeeds + + if("number" !in cgi.get) + assert(a == 10); // no value means you can't convert - give the default + + if(cgi.get["number"] == "twelve") + assert(a == 10); // conversion from string to int would fail, so we get the default + --- + + You can use an enum as an easy whitelist, too: + + --- + enum Operations { + add, remove, query + } + + auto op = cgi.request("op", Operations.query); + + if(cgi.get["op"] == "add") + assert(op == Operations.add); + if(cgi.get["op"] == "remove") + assert(op == Operations.remove); + if(cgi.get["op"] == "query") + assert(op == Operations.query); + + if(cgi.get["op"] == "random string") + assert(op == Operations.query); // the value can't be converted to the enum, so we get the default + --- + +/ + T request(T = string)(in string name, in T def = T.init) const nothrow { + try { + return + (name in post) ? to!T(post[name]) : + (name in get) ? to!T(get[name]) : + def; + } catch(Exception e) { return def; } + } + + /// Is the output already closed? + bool isClosed() const { + return closed; + } + + private SessionObject commandLineSessionObject; + + /++ + Gets a session object associated with the `cgi` request. You can use different type throughout your application. + +/ + Session!Data getSessionObject(Data)() { + if(testInProcess !is null) { + // test mode + auto obj = testInProcess.getSessionOverride(typeid(typeof(return))); + if(obj !is null) + return cast(typeof(return)) obj; + else { + auto o = new MockSession!Data(); + testInProcess.setSessionOverride(typeid(typeof(return)), o); + return o; + } + } else { + // FIXME: the changes are not printed out at the end! + if(_commandLineSession !is null) { + if(commandLineSessionObject is null) { + auto clso = new MockSession!Data(); + commandLineSessionObject = clso; + + + foreach(memberName; __traits(allMembers, Data)) { + if(auto str = memberName in _commandLineSession) + __traits(getMember, clso.store_, memberName) = to!(typeof(__traits(getMember, Data, memberName)))(*str); + } + } + + return cast(typeof(return)) commandLineSessionObject; + } + + // normal operation + return new BasicDataServerSession!Data(this); + } + } + + // if it is in test mode; triggers mock sessions. Used by CgiTester + version(with_breaking_cgi_features) + private CgiTester testInProcess; + + /* Hooks for redirecting input and output */ + private void delegate(const(ubyte)[]) rawDataOutput = null; + private void delegate() flushDelegate = null; + + /* This info is used when handling a more raw HTTP protocol */ + private bool nph; + private bool http10; + private bool closed; + private bool responseChunked = false; + + version(preserveData) // note: this can eat lots of memory; don't use unless you're sure you need it. + immutable(ubyte)[] originalPostData; + + /++ + This holds the posted body data if it has not been parsed into [post] and [postArray]. + + It is intended to be used for JSON and XML request content types, but also may be used + for other content types your application can handle. But it will NOT be populated + for content types application/x-www-form-urlencoded or multipart/form-data, since those are + parsed into the post and postArray members. + + Remember that anything beyond your `maxContentLength` param when setting up [GenericMain], etc., + will be discarded to the client with an error. This helps keep this array from being exploded in size + and consuming all your server's memory (though it may still be possible to eat excess ram from a concurrent + client in certain build modes.) + + History: + Added January 5, 2021 + Documented February 21, 2023 (dub v11.0) + +/ + public immutable string postBody; + alias postJson = postBody; // old name + + /++ + The content type header of the request. The [postBody] member may hold the actual data (see [postBody] for details). + + History: + Added January 26, 2024 (dub v11.4) + +/ + public immutable string requestContentType; + + /* Internal state flags */ + private bool outputtedResponseData; + private bool noCache = true; + + const(string[string]) environmentVariables; + + /** What follows is data gotten from the HTTP request. It is all fully immutable, + partially because it logically is (your code doesn't change what the user requested...) + and partially because I hate how bad programs in PHP change those superglobals to do + all kinds of hard to follow ugliness. I don't want that to ever happen in D. + + For some of these, you'll want to refer to the http or cgi specs for more details. + */ + immutable(string[string]) requestHeaders; /// All the raw headers in the request as name/value pairs. The name is stored as all lower case, but otherwise the same as it is in HTTP; words separated by dashes. For example, "cookie" or "accept-encoding". Many HTTP headers have specialized variables below for more convenience and static name checking; you should generally try to use them. + + immutable(char[]) host; /// The hostname in the request. If one program serves multiple domains, you can use this to differentiate between them. + immutable(char[]) origin; /// The origin header in the request, if present. Some HTML5 cross-domain apis set this and you should check it on those cross domain requests and websockets. + immutable(char[]) userAgent; /// The browser's user-agent string. Can be used to identify the browser. + immutable(char[]) pathInfo; /// This is any stuff sent after your program's name on the url, but before the query string. For example, suppose your program is named "app". If the user goes to site.com/app, pathInfo is empty. But, he can also go to site.com/app/some/sub/path; treating your program like a virtual folder. In this case, pathInfo == "/some/sub/path". + immutable(char[]) scriptName; /// The full base path of your program, as seen by the user. If your program is located at site.com/programs/apps, scriptName == "/programs/apps". + immutable(char[]) scriptFileName; /// The physical filename of your script + immutable(char[]) authorization; /// The full authorization string from the header, undigested. Useful for implementing auth schemes such as OAuth 1.0. Note that some web servers do not forward this to the app without taking extra steps. See requireBasicAuth's comment for more info. + immutable(char[]) accept; /// The HTTP accept header is the user agent telling what content types it is willing to accept. This is often */*; they accept everything, so it's not terribly useful. (The similar sounding Accept-Encoding header is handled automatically for chunking and gzipping. Simply set gzipResponse = true and cgi.d handles the details, zipping if the user's browser is willing to accept it.) + immutable(char[]) lastEventId; /// The HTML 5 draft includes an EventSource() object that connects to the server, and remains open to take a stream of events. My arsd.rtud module can help with the server side part of that. The Last-Event-Id http header is defined in the draft to help handle loss of connection. When the browser reconnects to you, it sets this header to the last event id it saw, so you can catch it up. This member has the contents of that header. + + immutable(RequestMethod) requestMethod; /// The HTTP request verb: GET, POST, etc. It is represented as an enum in cgi.d (which, like many enums, you can convert back to string with std.conv.to()). A HTTP GET is supposed to, according to the spec, not have side effects; a user can GET something over and over again and always have the same result. On all requests, the get[] and getArray[] members may be filled in. The post[] and postArray[] members are only filled in on POST methods. + immutable(char[]) queryString; /// The unparsed content of the request query string - the stuff after the ? in your URL. See get[] and getArray[] for a parse view of it. Sometimes, the unparsed string is useful though if you want a custom format of data up there (probably not a good idea, unless it is really simple, like "?username" perhaps.) + immutable(char[]) cookie; /// The unparsed content of the Cookie: header in the request. See also the cookies[string] member for a parsed view of the data. + /** The Referer header from the request. (It is misspelled in the HTTP spec, and thus the actual request and cgi specs too, but I spelled the word correctly here because that's sane. The spec's misspelling is an implementation detail.) It contains the site url that referred the user to your program; the site that linked to you, or if you're serving images, the site that has you as an image. Also, if you're in an iframe, the referrer is the site that is framing you. + + Important note: if the user copy/pastes your url, this is blank, and, just like with all other user data, their browsers can also lie to you. Don't rely on it for real security. + */ + immutable(char[]) referrer; + immutable(char[]) requestUri; /// The full url if the current request, excluding the protocol and host. requestUri == scriptName ~ pathInfo ~ (queryString.length ? "?" ~ queryString : ""); + + immutable(char[]) remoteAddress; /// The IP address of the user, as we see it. (Might not match the IP of the user's computer due to things like proxies and NAT.) + + immutable bool https; /// Was the request encrypted via https? + immutable int port; /// On what TCP port number did the server receive the request? + + /** Here come the parsed request variables - the things that come close to PHP's _GET, _POST, etc. superglobals in content. */ + + immutable(string[string]) get; /// The data from your query string in the url, only showing the last string of each name. If you want to handle multiple values with the same name, use getArray. This only works right if the query string is x-www-form-urlencoded; the default you see on the web with name=value pairs separated by the & character. + immutable(string[string]) post; /// The data from the request's body, on POST requests. It parses application/x-www-form-urlencoded data (used by most web requests, including typical forms), and multipart/form-data requests (used by file uploads on web forms) into the same container, so you can always access them the same way. It makes no attempt to parse other content types. If you want to accept an XML Post body (for a web api perhaps), you'll need to handle the raw data yourself. + immutable(string[string]) cookies; /// Separates out the cookie header into individual name/value pairs (which is how you set them!) + + /// added later + alias query = get; + + /** + Represents user uploaded files. + + When making a file upload form, be sure to follow the standard: set method="POST" and enctype="multipart/form-data" in your html
tag attributes. The key into this array is the name attribute on your input tag, just like with other post variables. See the comments on the UploadedFile struct for more information about the data inside, including important notes on max size and content location. + */ + immutable(UploadedFile[][string]) filesArray; + immutable(UploadedFile[string]) files; + + /// Use these if you expect multiple items submitted with the same name. btw, assert(get[name] is getArray[name][$-1); should pass. Same for post and cookies. + /// the order of the arrays is the order the data arrives + immutable(string[][string]) getArray; /// like get, but an array of values per name + immutable(string[][string]) postArray; /// ditto for post + immutable(string[][string]) cookiesArray; /// ditto for cookies + + private string[string] _commandLineSession; + + // convenience function for appending to a uri without extra ? + // matches the name and effect of javascript's location.search property + string search() const { + if(queryString.length) + return "?" ~ queryString; + return ""; + } + + // FIXME: what about multiple files with the same name? + private: + //RequestMethod _requestMethod; +} + +/// use this for testing or other isolated things when you want it to be no-ops +Cgi dummyCgi(Cgi.RequestMethod method = Cgi.RequestMethod.GET, string url = null, in ubyte[] data = null, void delegate(const(ubyte)[]) outputSink = null) { + // we want to ignore, not use stdout + if(outputSink is null) + outputSink = delegate void(const(ubyte)[]) { }; + + string[string] env; + env["REQUEST_METHOD"] = to!string(method); + env["CONTENT_LENGTH"] = to!string(data.length); + + auto cgi = new Cgi( + 0, + env, + { return data; }, + outputSink, + null); + + return cgi; +} + +/++ + A helper test class for request handler unittests. ++/ +version(with_breaking_cgi_features) +class CgiTester { + private { + SessionObject[TypeInfo] mockSessions; + SessionObject getSessionOverride(TypeInfo ti) { + if(auto o = ti in mockSessions) + return *o; + else + return null; + } + void setSessionOverride(TypeInfo ti, SessionObject so) { + mockSessions[ti] = so; + } + } + + /++ + Gets (and creates if necessary) a mock session object for this test. Note + it will be the same one used for any test operations through this CgiTester instance. + +/ + Session!Data getSessionObject(Data)() { + auto obj = getSessionOverride(typeid(typeof(return))); + if(obj !is null) + return cast(typeof(return)) obj; + else { + auto o = new MockSession!Data(); + setSessionOverride(typeid(typeof(return)), o); + return o; + } + } + + /++ + Pass a reference to your request handler when creating the tester. + +/ + this(void function(Cgi) requestHandler) { + this.requestHandler = requestHandler; + } + + /++ + You can check response information with these methods after you call the request handler. + +/ + struct Response { + int code; + string[string] headers; + string responseText; + ubyte[] responseBody; + } + + /++ + Executes a test request on your request handler, and returns the response. + + Params: + url = The URL to test. Should be an absolute path, but excluding domain. e.g. `"/test"`. + args = additional arguments. Same format as cgi's command line handler. + +/ + Response GET(string url, string[] args = null) { + return executeTest("GET", url, args); + } + /// ditto + Response POST(string url, string[] args = null) { + return executeTest("POST", url, args); + } + + /// ditto + Response executeTest(string method, string url, string[] args) { + ubyte[] outputtedRawData; + void outputSink(const(ubyte)[] data) { + outputtedRawData ~= data; + } + auto cgi = new Cgi(["test", method, url] ~ args, &outputSink); + cgi.testInProcess = this; + scope(exit) cgi.dispose(); + + requestHandler(cgi); + + cgi.close(); + + Response response; + + if(outputtedRawData.length) { + enum LINE = "\r\n"; + + auto idx = outputtedRawData.locationOf(LINE ~ LINE); + assert(idx != -1, to!string(outputtedRawData)); + auto headers = cast(string) outputtedRawData[0 .. idx]; + response.code = 200; + while(headers.length) { + auto i = headers.locationOf(LINE); + if(i == -1) i = cast(int) headers.length; + + auto header = headers[0 .. i]; + + auto c = header.locationOf(":"); + if(c != -1) { + auto name = header[0 .. c]; + auto value = header[c + 2 ..$]; + + if(name == "Status") + response.code = value[0 .. value.locationOf(" ")].to!int; + + response.headers[name] = value; + } else { + assert(0); + } + + if(i != headers.length) + i += 2; + headers = headers[i .. $]; + } + response.responseBody = outputtedRawData[idx + 4 .. $]; + response.responseText = cast(string) response.responseBody; + } + + return response; + } + + private void function(Cgi) requestHandler; +} + + +// should this be a separate module? Probably, but that's a hassle. + +/// Makes a data:// uri that can be used as links in most newer browsers (IE8+). +string makeDataUrl(string mimeType, in void[] data) { + auto data64 = Base64.encode(cast(const(ubyte[])) data); + return "data:" ~ mimeType ~ ";base64," ~ assumeUnique(data64); +} + +// FIXME: I don't think this class correctly decodes/encodes the individual parts +/// Represents a url that can be broken down or built up through properties +struct Uri { + alias toString this; // blargh idk a url really is a string, but should it be implicit? + + // scheme//userinfo@host:port/path?query#fragment + + string scheme; /// e.g. "http" in "http://example.com/" + string userinfo; /// the username (and possibly a password) in the uri + string host; /// the domain name + int port; /// port number, if given. Will be zero if a port was not explicitly given + string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" + string query; /// the stuff after the ? in a uri + string fragment; /// the stuff after the # in a uri. + + // idk if i want to keep these, since the functions they wrap are used many, many, many times in existing code, so this is either an unnecessary alias or a gratuitous break of compatibility + // the decode ones need to keep different names anyway because we can't overload on return values... + static string encode(string s) { return encodeUriComponent(s); } + static string encode(string[string] s) { return encodeVariables(s); } + static string encode(string[][string] s) { return encodeVariables(s); } + + /// Breaks down a uri string to its components + this(string uri) { + reparse(uri); + } + + private void reparse(string uri) { + // from RFC 3986 + // the ctRegex triples the compile time and makes ugly errors for no real benefit + // it was a nice experiment but just not worth it. + // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; + /* + Captures: + 0 = whole url + 1 = scheme, with : + 2 = scheme, no : + 3 = authority, with // + 4 = authority, no // + 5 = path + 6 = query string, with ? + 7 = query string, no ? + 8 = anchor, with # + 9 = anchor, no # + */ + // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! + // instead, I will DIY and cut that down to 0.6s on the same computer. + /* + + Note that authority is + user:password@domain:port + where the user:password@ part is optional, and the :port is optional. + + Regex translation: + + Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. + Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. + Path cannot have any ? or # in it. It is optional. + Query must start with ? and must not have # in it. It is optional. + Anchor must start with # and can have anything else in it to end of string. It is optional. + */ + + this = Uri.init; // reset all state + + // empty uri = nothing special + if(uri.length == 0) { + return; + } + + size_t idx; + + scheme_loop: foreach(char c; uri[idx .. $]) { + switch(c) { + case ':': + case '/': + case '?': + case '#': + break scheme_loop; + default: + } + idx++; + } + + if(idx == 0 && uri[idx] == ':') { + // this is actually a path! we skip way ahead + goto path_loop; + } + + if(idx == uri.length) { + // the whole thing is a path, apparently + path = uri; + return; + } + + if(idx > 0 && uri[idx] == ':') { + scheme = uri[0 .. idx]; + idx++; + } else { + // we need to rewind; it found a / but no :, so the whole thing is prolly a path... + idx = 0; + } + + if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { + // we have an authority.... + idx += 2; + + auto authority_start = idx; + authority_loop: foreach(char c; uri[idx .. $]) { + switch(c) { + case '/': + case '?': + case '#': + break authority_loop; + default: + } + idx++; + } + + auto authority = uri[authority_start .. idx]; + + auto idx2 = authority.indexOf("@"); + if(idx2 != -1) { + userinfo = authority[0 .. idx2]; + authority = authority[idx2 + 1 .. $]; + } + + if(authority.length && authority[0] == '[') { + // ipv6 address special casing + idx2 = authority.indexOf(']'); + if(idx2 != -1) { + auto end = authority[idx2 + 1 .. $]; + if(end.length && end[0] == ':') + idx2 = idx2 + 1; + else + idx2 = -1; + } + } else { + idx2 = authority.indexOf(":"); + } + + if(idx2 == -1) { + port = 0; // 0 means not specified; we should use the default for the scheme + host = authority; + } else { + host = authority[0 .. idx2]; + if(idx2 + 1 < authority.length) + port = to!int(authority[idx2 + 1 .. $]); + else + port = 0; + } + } + + path_loop: + auto path_start = idx; + + foreach(char c; uri[idx .. $]) { + if(c == '?' || c == '#') + break; + idx++; + } + + path = uri[path_start .. idx]; + + if(idx == uri.length) + return; // nothing more to examine... + + if(uri[idx] == '?') { + idx++; + auto query_start = idx; + foreach(char c; uri[idx .. $]) { + if(c == '#') + break; + idx++; + } + query = uri[query_start .. idx]; + } + + if(idx < uri.length && uri[idx] == '#') { + idx++; + fragment = uri[idx .. $]; + } + + // uriInvalidated = false; + } + + private string rebuildUri() const { + string ret; + if(scheme.length) + ret ~= scheme ~ ":"; + if(userinfo.length || host.length) + ret ~= "//"; + if(userinfo.length) + ret ~= userinfo ~ "@"; + if(host.length) + ret ~= host; + if(port) + ret ~= ":" ~ to!string(port); + + ret ~= path; + + if(query.length) + ret ~= "?" ~ query; + + if(fragment.length) + ret ~= "#" ~ fragment; + + // uri = ret; + // uriInvalidated = false; + return ret; + } + + /// Converts the broken down parts back into a complete string + string toString() const { + // if(uriInvalidated) + return rebuildUri(); + } + + /// Returns a new absolute Uri given a base. It treats this one as + /// relative where possible, but absolute if not. (If protocol, domain, or + /// other info is not set, the new one inherits it from the base.) + /// + /// Browsers use a function like this to figure out links in html. + Uri basedOn(in Uri baseUrl) const { + Uri n = this; // copies + if(n.scheme == "data") + return n; + // n.uriInvalidated = true; // make sure we regenerate... + + // userinfo is not inherited... is this wrong? + + // if anything is given in the existing url, we don't use the base anymore. + if(n.scheme.empty) { + n.scheme = baseUrl.scheme; + if(n.host.empty) { + n.host = baseUrl.host; + if(n.port == 0) { + n.port = baseUrl.port; + if(n.path.length > 0 && n.path[0] != '/') { + auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; + if(b.length == 0) + b = "/"; + n.path = b ~ n.path; + } else if(n.path.length == 0) { + n.path = baseUrl.path; + } + } + } + } + + n.removeDots(); + + return n; + } + + void removeDots() { + auto parts = this.path.split("/"); + string[] toKeep; + foreach(part; parts) { + if(part == ".") { + continue; + } else if(part == "..") { + //if(toKeep.length > 1) + toKeep = toKeep[0 .. $-1]; + //else + //toKeep = [""]; + continue; + } else { + //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) + //continue; // skip a `//` situation + toKeep ~= part; + } + } + + auto path = toKeep.join("/"); + if(path.length && path[0] != '/') + path = "/" ~ path; + + this.path = path; + } + + unittest { + auto uri = Uri("test.html"); + assert(uri.path == "test.html"); + uri = Uri("path/1/lol"); + assert(uri.path == "path/1/lol"); + uri = Uri("http://me@example.com"); + assert(uri.scheme == "http"); + assert(uri.userinfo == "me"); + assert(uri.host == "example.com"); + uri = Uri("http://example.com/#a"); + assert(uri.scheme == "http"); + assert(uri.host == "example.com"); + assert(uri.fragment == "a"); + uri = Uri("#foo"); + assert(uri.fragment == "foo"); + uri = Uri("?lol"); + assert(uri.query == "lol"); + uri = Uri("#foo?lol"); + assert(uri.fragment == "foo?lol"); + uri = Uri("?lol#foo"); + assert(uri.fragment == "foo"); + assert(uri.query == "lol"); + + uri = Uri("http://127.0.0.1/"); + assert(uri.host == "127.0.0.1"); + assert(uri.port == 0); + + uri = Uri("http://127.0.0.1:123/"); + assert(uri.host == "127.0.0.1"); + assert(uri.port == 123); + + uri = Uri("http://[ff:ff::0]/"); + assert(uri.host == "[ff:ff::0]"); + + uri = Uri("http://[ff:ff::0]:123/"); + assert(uri.host == "[ff:ff::0]"); + assert(uri.port == 123); + } + + // This can sometimes be a big pain in the butt for me, so lots of copy/paste here to cover + // the possibilities. + unittest { + auto url = Uri("cool.html"); // checking relative links + + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/cool.html"); + + url = Uri("/something/cool.html"); // same server, different path + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/something/cool.html"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com/something/cool.html"); + + url = Uri("?query=answer"); // same path. server, protocol, and port, just different query string and fragment + assert(url.basedOn(Uri("http://test.com/what/test.html")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("https://test.com/what/test.html")) == "https://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/")) == "http://test.com/what/?query=answer"); + assert(url.basedOn(Uri("http://test.com/")) == "http://test.com/?query=answer"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com/what/test.html?a=b&c=d#what")) == "http://test.com/what/test.html?query=answer"); + assert(url.basedOn(Uri("http://test.com")) == "http://test.com?query=answer"); + + url = Uri("/test/bar"); + assert(Uri("./").basedOn(url) == "/test/", Uri("./").basedOn(url)); + assert(Uri("../").basedOn(url) == "/"); + + url = Uri("http://example.com/"); + assert(Uri("../foo").basedOn(url) == "http://example.com/foo"); + + //auto uriBefore = url; + url = Uri("#anchor"); // everything should remain the same except the anchor + //uriBefore.anchor = "anchor"); + //assert(url == uriBefore); + + url = Uri("//example.com"); // same protocol, but different server. the path here should be blank. + + url = Uri("//example.com/example.html"); // same protocol, but different server and path + + url = Uri("http://example.com/test.html"); // completely absolute link should never be modified + + url = Uri("http://example.com"); // completely absolute link should never be modified, even if it has no path + + // FIXME: add something for port too + } + + // these are like javascript's location.search and location.hash + string search() const { + return query.length ? ("?" ~ query) : ""; + } + string hash() const { + return fragment.length ? ("#" ~ fragment) : ""; + } +} + + +/* + for session, see web.d +*/ + +/// breaks down a url encoded string +string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) { + auto vars = data.split(separator); + string[][string] _get; + foreach(var; vars) { + auto equal = var.indexOf("="); + string name; + string value; + if(equal == -1) { + name = decodeUriComponent(var); + value = ""; + } else { + //_get[decodeUriComponent(var[0..equal])] ~= decodeUriComponent(var[equal + 1 .. $].replace("+", " ")); + // stupid + -> space conversion. + name = decodeUriComponent(var[0..equal].replace("+", " ")); + value = decodeUriComponent(var[equal + 1 .. $].replace("+", " ")); + } + + _get[name] ~= value; + if(namesInOrder) + (*namesInOrder) ~= name; + if(valuesInOrder) + (*valuesInOrder) ~= value; + } + return _get; +} + +/// breaks down a url encoded string, but only returns the last value of any array +string[string] decodeVariablesSingle(string data) { + string[string] va; + auto varArray = decodeVariables(data); + foreach(k, v; varArray) + va[k] = v[$-1]; + + return va; +} + +/// url encodes the whole string +string encodeVariables(in string[string] data) { + string ret; + + bool outputted = false; + foreach(k, v; data) { + if(outputted) + ret ~= "&"; + else + outputted = true; + + ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v); + } + + return ret; +} + +/// url encodes a whole string +string encodeVariables(in string[][string] data) { + string ret; + + bool outputted = false; + foreach(k, arr; data) { + foreach(v; arr) { + if(outputted) + ret ~= "&"; + else + outputted = true; + ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v); + } + } + + return ret; +} + +/// Encodes all but the explicitly unreserved characters per rfc 3986 +/// Alphanumeric and -_.~ are the only ones left unencoded +/// name is borrowed from php +string rawurlencode(in char[] data) { + string ret; + ret.reserve(data.length * 2); + foreach(char c; data) { + if( + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~') + { + ret ~= c; + } else { + ret ~= '%'; + // since we iterate on char, this should give us the octets of the full utf8 string + ret ~= toHexUpper(c); + } + } + + return ret; +} + + +// http helper functions + +// for chunked responses (which embedded http does whenever possible) +version(none) // this is moved up above to avoid making a copy of the data +const(ubyte)[] makeChunk(const(ubyte)[] data) { + const(ubyte)[] ret; + + ret = cast(const(ubyte)[]) toHex(data.length); + ret ~= cast(const(ubyte)[]) "\r\n"; + ret ~= data; + ret ~= cast(const(ubyte)[]) "\r\n"; + + return ret; +} + +string toHex(long num) { + string ret; + while(num) { + int v = num % 16; + num /= 16; + char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'a'); + ret ~= d; + } + + return to!string(array(ret.retro)); +} + +string toHexUpper(long num) { + string ret; + while(num) { + int v = num % 16; + num /= 16; + char d = cast(char) ((v < 10) ? v + '0' : (v-10) + 'A'); + ret ~= d; + } + + if(ret.length == 1) + ret ~= "0"; // url encoding requires two digits and that's what this function is used for... + + return to!string(array(ret.retro)); +} + + +// the generic mixins + +/++ + Use this instead of writing your own main + + It ultimately calls [cgiMainImpl] which creates a [RequestServer] for you. ++/ +mixin template GenericMain(alias fun, long maxContentLength = defaultMaxContentLength) { + mixin CustomCgiMain!(Cgi, fun, maxContentLength); +} + +/++ + Boilerplate mixin for a main function that uses the [dispatcher] function. + + You can send `typeof(null)` as the `Presenter` argument to use a generic one. + + History: + Added July 9, 2021 ++/ +mixin template DispatcherMain(Presenter, DispatcherArgs...) { + /// forwards to [CustomCgiDispatcherMain] with default args + mixin CustomCgiDispatcherMain!(Cgi, defaultMaxContentLength, Presenter, DispatcherArgs); +} + +/// ditto +mixin template DispatcherMain(DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { + class GenericPresenter : WebPresenter!GenericPresenter {} + mixin DispatcherMain!(GenericPresenter, DispatcherArgs); +} + +/++ + Allows for a generic [DispatcherMain] with custom arguments. Note you can use [defaultMaxContentLength] as the second argument if you like. + + History: + Added May 13, 2023 (dub v11.0) ++/ +mixin template CustomCgiDispatcherMain(CustomCgi, size_t maxContentLength, Presenter, DispatcherArgs...) { + /++ + Handler to the generated presenter you can use from your objects, etc. + +/ + Presenter activePresenter; + + /++ + Request handler that creates the presenter then forwards to the [dispatcher] function. + Renders 404 if the dispatcher did not handle the request. + + Will automatically serve the presenter.style and presenter.script as "style.css" and "script.js" + +/ + void handler(Cgi cgi) { + auto presenter = new Presenter; + activePresenter = presenter; + scope(exit) activePresenter = null; + + if(cgi.pathInfo.length == 0) { + cgi.setResponseLocation(cgi.scriptName ~ "/"); + return; + } + + if(cgi.dispatcher!DispatcherArgs(presenter)) + return; + + switch(cgi.pathInfo) { + case "/style.css": + cgi.setCache(true); + cgi.setResponseContentType("text/css"); + cgi.write(presenter.style(), true); + break; + case "/script.js": + cgi.setCache(true); + cgi.setResponseContentType("application/javascript"); + cgi.write(presenter.script(), true); + break; + default: + presenter.renderBasicError(cgi, 404); + } + } + mixin CustomCgiMain!(CustomCgi, handler, maxContentLength); +} + +/// ditto +mixin template CustomCgiDispatcherMain(CustomCgi, size_t maxContentLength, DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { + class GenericPresenter : WebPresenter!GenericPresenter {} + mixin CustomCgiDispatcherMain!(CustomCgi, maxContentLength, GenericPresenter, DispatcherArgs); + +} + +private string simpleHtmlEncode(string s) { + return s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\n", "
\n"); +} + +string messageFromException(Throwable t) { + string message; + if(t !is null) { + debug message = t.toString(); + else message = "An unexpected error has occurred."; + } else { + message = "Unknown error"; + } + return message; +} + +string plainHttpError(bool isCgi, string type, Throwable t) { + auto message = messageFromException(t); + message = simpleHtmlEncode(message); + + return format("%s %s\r\nContent-Length: %s\r\nConnection: close\r\n\r\n%s", + isCgi ? "Status:" : "HTTP/1.1", + type, message.length, message); +} + +// returns true if we were able to recover reasonably +bool handleException(Cgi cgi, Throwable t) { + if(cgi.isClosed) { + // if the channel has been explicitly closed, we can't handle it here + return true; + } + + if(cgi.outputtedResponseData) { + // the headers are sent, but the channel is open... since it closes if all was sent, we can append an error message here. + return false; // but I don't want to, since I don't know what condition the output is in; I don't want to inject something (nor check the content-type for that matter. So we say it was not a clean handling. + } else { + // no headers are sent, we can send a full blown error and recover + cgi.setCache(false); + cgi.setResponseContentType("text/html"); + cgi.setResponseLocation(null); // cancel the redirect + cgi.setResponseStatus("500 Internal Server Error"); + cgi.write(simpleHtmlEncode(messageFromException(t))); + cgi.close(); + return true; + } +} + +bool isCgiRequestMethod(string s) { + s = s.toUpper(); + if(s == "COMMANDLINE") + return true; + foreach(member; __traits(allMembers, Cgi.RequestMethod)) + if(s == member) + return true; + return false; +} + +/// If you want to use a subclass of Cgi with generic main, use this mixin. +mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { + // kinda hacky - the T... is passed to Cgi's constructor in standard cgi mode, and ignored elsewhere + void main(string[] args) { + cgiMainImpl!(fun, CustomCgi, maxContentLength)(args); + } +} + +version(embedded_httpd_processes) + __gshared int processPoolSize = 8; + +// Returns true if run. You should exit the program after that. +bool tryAddonServers(string[] args) { + if(args.length > 1) { + // run the special separate processes if needed + switch(args[1]) { + case "--websocket-server": + version(with_addon_servers) + websocketServers[args[2]](args[3 .. $]); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--websocket-servers": + import core.demangle; + version(with_addon_servers_connections) + foreach(k, v; websocketServers) + writeln(k, "\t", demangle(k)); + return true; + case "--session-server": + version(with_addon_servers) + runSessionServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--event-server": + version(with_addon_servers) + runEventServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--timer-server": + version(with_addon_servers) + runTimerServer(); + else + printf("Add-on servers not compiled in.\n"); + return true; + case "--timed-jobs": + import core.demangle; + version(with_addon_servers_connections) + foreach(k, v; scheduledJobHandlers) + writeln(k, "\t", demangle(k)); + return true; + case "--timed-job": + scheduledJobHandlers[args[2]](args[3 .. $]); + return true; + default: + // intentionally blank - do nothing and carry on to run normally + } + } + return false; +} + +/// Tries to simulate a request from the command line. Returns true if it does, false if it didn't find the args. +bool trySimulatedRequest(alias fun, CustomCgi = Cgi)(string[] args) if(is(CustomCgi : Cgi)) { + // we support command line thing for easy testing everywhere + // it needs to be called ./app method uri [other args...] + if(args.length >= 3 && isCgiRequestMethod(args[1])) { + Cgi cgi = new CustomCgi(args); + scope(exit) cgi.dispose(); + try { + fun(cgi); + cgi.close(); + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } + writeln(); // just to put a blank line before the prompt cuz it annoys me + // FIXME: put in some footers to show what changes happened in the session + // could make the MockSession be some kind of ReflectableSessionObject or something + return true; + } + return false; +} + +/++ + A server control and configuration struct, as a potential alternative to calling [GenericMain] or [cgiMainImpl]. See the source of [cgiMainImpl] for a complete, up-to-date, example of how it is used internally. + + As of version 11 (released August 2023), you can also make things like this: + + --- + // listens on both a unix domain socket called `foo` and on the loopback interfaces port 8080 + RequestServer server = RequestServer(["http://unix:foo", "http://localhost:8080"]); + + // can also: + // RequestServer server = RequestServer(0); // listen on an OS-provided port on all interfaces + + // NOT IMPLEMENTED YET + // server.initialize(); // explicit initialization will populate any "any port" things and throw if a bind failed + + foreach(listenSpec; server.listenSpecs) { + // you can check what it actually bound to here and see your assigned ports + } + + // NOT IMPLEMENTED YET + // server.start!handler(); // starts and runs in the arsd.core event loop + + server.serve!handler(); // blocks the thread until the server exits + --- + + History: + Added Sept 26, 2020 (release version 8.5). + + The `listenSpec` member was added July 31, 2023. ++/ +struct RequestServer { + /++ + Sets the host and port the server will listen on. This is semi-deprecated; the new (as of July 31, 2023) [listenSpec] parameter obsoletes these. You cannot use both together; the listeningHost and listeningPort are ONLY used if listenSpec is null. + +/ + string listeningHost = defaultListeningHost(); + /// ditto + ushort listeningPort = defaultListeningPort(); + + static struct ListenSpec { + enum Protocol { + http, + https, + scgi + } + Protocol protocol; + + enum AddressType { + ip, + unix, + abstract_ + } + AddressType addressType; + + string address; + ushort port; + } + + /++ + The array of addresses you want to listen on. The format looks like a url but has a few differences. + + This ONLY works on embedded_httpd_threads, embedded_httpd_hybrid, and scgi builds at this time. + + `http://localhost:8080` + + `http://unix:filename/here` + + `scgi://abstract:/name/here` + + `http://[::1]:4444` + + Note that IPv6 addresses must be enclosed in brackets. If you want to listen on an interface called `unix` or `abstract`, contact me, that is not supported but I could add some kind of escape mechanism. + + If you leave off the protocol, it assumes the default based on compile flags. If you only give a number, it is assumed to be a port on any tcp interface. + + `localhost:8080` serves the default protocol. + + `8080` or `:8080` assumes default protocol on localhost. + + The protocols can be `http:`, `https:`, and `scgi:`. Original `cgi` is not supported with this, since it is transactional with a single process. + + Valid hosts are an IPv4 address (with a mandatory port), an IPv6 address (with a mandatory port), just a port alone, `unix:/path/to/unix/socket` (which may be a relative path without a leading slash), or `abstract:/path/to/linux/abstract/namespace`. + + `http://unix:foo` will serve http over the unix domain socket named `foo` in the current working directory. + + $(PITFALL + If you set this to anything non-null (including a non-null, zero-length array) any `listenSpec` entries, [listeningHost] and [listeningPort] are ignored. + ) + + Bugs: + The implementation currently ignores the protocol spec in favor of the default compiled in option. + + History: + Added July 31, 2023 (dub v11.0) + +/ + string[] listenSpec; + + /++ + Uses a fork() call, if available, to provide additional crash resiliency and possibly improved performance. On the + other hand, if you fork, you must not assume any memory is shared between requests (you shouldn't be anyway though! But + if you have to, you probably want to set this to false and use an explicit threaded server with [serveEmbeddedHttp]) and + [stop] may not work as well. + + History: + Added August 12, 2022 (dub v10.9). Previously, this was only configurable through the `-version=cgi_no_fork` + argument to dmd. That version still defines the value of `cgi_use_fork_default`, used to initialize this, for + compatibility. + +/ + bool useFork = cgi_use_fork_default; + + /++ + Determines the number of worker threads to spawn per process, for server modes that use worker threads. 0 will use a + default based on the number of cpus modified by the server mode. + + History: + Added August 12, 2022 (dub v10.9) + +/ + int numberOfThreads = 0; + + /++ + Creates a server configured to listen to multiple URLs. + + History: + Added July 31, 2023 (dub v11.0) + +/ + this(string[] listenTo) { + this.listenSpec = listenTo; + } + + /// Creates a server object configured to listen on a single host and port. + this(string defaultHost, ushort defaultPort) { + this.listeningHost = defaultHost; + this.listeningPort = defaultPort; + } + + /// ditto + this(ushort defaultPort) { + listeningPort = defaultPort; + } + + /++ + Reads the command line arguments into the values here. + + Possible arguments are `--listen` (can appear multiple times), `--listening-host`, `--listening-port` (or `--port`), `--uid`, and `--gid`. + + Please note you cannot combine `--listen` with `--listening-host` or `--listening-port` / `--port`. Use one or the other style. + +/ + void configureFromCommandLine(string[] args) { + bool portOrHostFound = false; + + bool foundPort = false; + bool foundHost = false; + bool foundUid = false; + bool foundGid = false; + bool foundListen = false; + foreach(arg; args) { + if(foundPort) { + listeningPort = to!ushort(arg); + portOrHostFound = true; + foundPort = false; + continue; + } + if(foundHost) { + listeningHost = arg; + portOrHostFound = true; + foundHost = false; + continue; + } + if(foundUid) { + privilegesDropToUid = to!uid_t(arg); + foundUid = false; + continue; + } + if(foundGid) { + privilegesDropToGid = to!gid_t(arg); + foundGid = false; + continue; + } + if(foundListen) { + this.listenSpec ~= arg; + foundListen = false; + continue; + } + if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") + foundHost = true; + else if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") + foundPort = true; + else if(arg == "--uid") + foundUid = true; + else if(arg == "--gid") + foundGid = true; + else if(arg == "--listen") + foundListen = true; + } + + if(portOrHostFound && listenSpec.length) { + throw new Exception("You passed both a --listening-host or --listening-port and a --listen argument. You should fix your script to ONLY use --listen arguments."); + } + } + + version(Windows) { + private alias uid_t = int; + private alias gid_t = int; + } + + /// user (uid) to drop privileges to + /// 0 … do nothing + uid_t privilegesDropToUid = 0; + /// group (gid) to drop privileges to + /// 0 … do nothing + gid_t privilegesDropToGid = 0; + + private void dropPrivileges() { + version(Posix) { + import core.sys.posix.unistd; + + if (privilegesDropToGid != 0 && setgid(privilegesDropToGid) != 0) + throw new Exception("Dropping privileges via setgid() failed."); + + if (privilegesDropToUid != 0 && setuid(privilegesDropToUid) != 0) + throw new Exception("Dropping privileges via setuid() failed."); + } + else { + // FIXME: Windows? + //pragma(msg, "Dropping privileges is not implemented for this platform"); + } + + // done, set zero + privilegesDropToGid = 0; + privilegesDropToUid = 0; + } + + /++ + Serves a single HTTP request on this thread, with an embedded server, then stops. Designed for cases like embedded oauth responders + + History: + Added Oct 10, 2020. + Example: + + --- + import arsd.cgi; + void main() { + RequestServer server = RequestServer("127.0.0.1", 6789); + string oauthCode; + string oauthScope; + server.serveHttpOnce!((cgi) { + oauthCode = cgi.request("code"); + oauthScope = cgi.request("scope"); + cgi.write("Thank you, please return to the application."); + }); + // use the code and scope given + } + --- + +/ + void serveHttpOnce(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + import std.socket; + + bool tcp; + void delegate() cleanup; + auto socket = startListening(listeningHost, listeningPort, tcp, cleanup, 1, &dropPrivileges); + auto connection = socket.accept(); + doThreadHttpConnectionGuts!(CustomCgi, fun, true)(connection); + + if(cleanup) + cleanup(); + } + + /++ + Starts serving requests according to the current configuration. + +/ + void serve(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + version(netman_httpd) { + // Obsolete! + + import arsd.httpd; + // what about forwarding the other constructor args? + // this probably needs a whole redoing... + serveHttp!CustomCgi(&fun, listeningPort);//5005); + return; + } else + version(embedded_httpd_processes) { + serveEmbeddedHttpdProcesses!(fun, CustomCgi)(this); + } else + version(embedded_httpd_threads) { + serveEmbeddedHttp!(fun, CustomCgi, maxContentLength)(); + } else + version(scgi) { + serveScgi!(fun, CustomCgi, maxContentLength)(); + } else + version(fastcgi) { + serveFastCgi!(fun, CustomCgi, maxContentLength)(this); + } else + version(stdio_http) { + serveSingleHttpConnectionOnStdio!(fun, CustomCgi, maxContentLength)(); + } else { + //version=plain_cgi; + handleCgiRequest!(fun, CustomCgi, maxContentLength)(); + } + } + + /++ + Runs the embedded HTTP thread server specifically, regardless of which build configuration you have. + + If you want the forking worker process server, you do need to compile with the embedded_httpd_processes config though. + +/ + void serveEmbeddedHttp(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(ThisFor!fun _this) { + globalStopFlag = false; + static if(__traits(isStaticFunction, fun)) + alias funToUse = fun; + else + void funToUse(CustomCgi cgi) { + static if(__VERSION__ > 2097) + __traits(child, _this, fun)(cgi); + else static assert(0, "Not implemented in your compiler version!"); + } + auto manager = this.listenSpec is null ? + new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, funToUse), null, useFork, numberOfThreads) : + new ListeningConnectionManager(this.listenSpec, &doThreadHttpConnection!(CustomCgi, funToUse), null, useFork, numberOfThreads); + manager.listen(); + } + + /++ + Runs the embedded SCGI server specifically, regardless of which build configuration you have. + +/ + void serveScgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + globalStopFlag = false; + auto manager = this.listenSpec is null ? + new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength), null, useFork, numberOfThreads) : + new ListeningConnectionManager(this.listenSpec, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength), null, useFork, numberOfThreads); + manager.listen(); + } + + /++ + Serves a single "connection", but the connection is spoken on stdin and stdout instead of on a socket. + + Intended for cases like working from systemd, like discussed here: [https://forum.dlang.org/post/avmkfdiitirnrenzljwc@forum.dlang.org] + + History: + Added May 29, 2021 + +/ + void serveSingleHttpConnectionOnStdio(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + doThreadHttpConnectionGuts!(CustomCgi, fun, true)(new FakeSocketForStdin()); + } + + /++ + The [stop] function sets a flag that request handlers can (and should) check periodically. If a handler doesn't + respond to this flag, the library will force the issue. This determines when and how the issue will be forced. + +/ + enum ForceStop { + /++ + Stops accepting new requests, but lets ones already in the queue start and complete before exiting. + +/ + afterQueuedRequestsComplete, + /++ + Finishes requests already started their handlers, but drops any others in the queue. Streaming handlers + should cooperate and exit gracefully, but if they don't, it will continue waiting for them. + +/ + afterCurrentRequestsComplete, + /++ + Partial response writes will throw an exception, cancelling any streaming response, but complete + writes will continue to process. Request handlers that respect the stop token will also gracefully cancel. + +/ + cancelStreamingRequestsEarly, + /++ + All writes will throw. + +/ + cancelAllRequestsEarly, + /++ + Use OS facilities to forcibly kill running threads. The server process will be in an undefined state after this call (if this call ever returns). + +/ + forciblyTerminate, + } + + version(embedded_httpd_processes) {} else + /++ + Stops serving after the current requests are completed. + + Bugs: + Not implemented on version=embedded_httpd_processes, version=fastcgi on any system, or embedded_httpd on Windows (it does work on embedded_httpd_hybrid + on Windows however). Only partially implemented on non-Linux posix systems. + + You might also try SIGINT perhaps. + + The stopPriority is not yet fully implemented. + +/ + static void stop(ForceStop stopPriority = ForceStop.afterCurrentRequestsComplete) { + globalStopFlag = true; + + version(Posix) { + if(cancelfd > 0) { + ulong a = 1; + core.sys.posix.unistd.write(cancelfd, &a, a.sizeof); + } + } + version(Windows) { + if(iocp) { + foreach(i; 0 .. 16) // FIXME + PostQueuedCompletionStatus(iocp, 0, cast(ULONG_PTR) null, null); + } + } + } +} + +class AuthorizationRequiredException : Exception { + string type; + string realm; + this(string type, string realm, string file, size_t line) { + this.type = type; + this.realm = realm; + + super("Authorization Required", file, line); + } +} + +private alias AliasSeq(T...) = T; + +version(with_breaking_cgi_features) +mixin(q{ + template ThisFor(alias t) { + static if(__traits(isStaticFunction, t)) { + alias ThisFor = AliasSeq!(); + } else { + alias ThisFor = __traits(parent, t); + } + } +}); +else + alias ThisFor(alias t) = AliasSeq!(); + +private __gshared bool globalStopFlag = false; + +version(embedded_httpd_processes) +void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer params) { + import core.sys.posix.unistd; + import core.sys.posix.sys.socket; + import core.sys.posix.netinet.in_; + //import std.c.linux.socket; + + int sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket"); + + cloexec(sock); + + { + + sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(params.listeningPort); + auto lh = params.listeningHost; + if(lh.length) { + if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) + throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); + } else + addr.sin_addr.s_addr = INADDR_ANY; + + // HACKISH + int on = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); + // end hack + + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + close(sock); + throw new Exception("bind"); + } + + // FIXME: if this queue is full, it will just ignore it + // and wait for the client to retransmit it. This is an + // obnoxious timeout condition there. + if(sock.listen(128) == -1) { + close(sock); + throw new Exception("listen"); + } + params.dropPrivileges(); + } + + version(embedded_httpd_processes_accept_after_fork) {} else { + int pipeReadFd; + int pipeWriteFd; + + { + int[2] pipeFd; + if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pipeFd)) { + import core.stdc.errno; + throw new Exception("pipe failed " ~ to!string(errno)); + } + + pipeReadFd = pipeFd[0]; + pipeWriteFd = pipeFd[1]; + } + } + + + int processCount; + pid_t newPid; + reopen: + while(processCount < processPoolSize) { + newPid = fork(); + if(newPid == 0) { + // start serving on the socket + //ubyte[4096] backingBuffer; + for(;;) { + bool closeConnection; + uint i; + sockaddr addr; + i = addr.sizeof; + version(embedded_httpd_processes_accept_after_fork) { + int s = accept(sock, &addr, &i); + int opt = 1; + import core.sys.posix.netinet.tcp; + // the Cgi class does internal buffering, so disabling this + // helps with latency in many cases... + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + cloexec(s); + } else { + int s; + auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s); + if(readret != s.sizeof) { + import core.stdc.errno; + throw new Exception("pipe read failed " ~ to!string(errno)); + } + + //writeln("process ", getpid(), " got socket ", s); + } + + try { + + if(s == -1) + throw new Exception("accept"); + + scope(failure) close(s); + //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; + auto ir = new BufferedInputRange(s); + //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); + + while(!ir.empty) { + //ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = cast(CgiConnectionHandle) s; + // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. + if(processPoolSize <= 1) + closeConnection = true; + //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); + } catch(HttpVersionNotSupportedException he) { + sendAll(ir.source, plainHttpError(false, "505 HTTP Version Not Supported", he)); + closeConnection = true; + break; + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto estr = t.toString(); + stderr.rawWrite(estr); + stderr.rawWrite("\n"); + } else + stderr.writeln(t.toString()); + sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + if(cgi.websocketMode) + closeConnection = true; + + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } catch(ConnectionException ce) { + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto estr = t.toString(); + stderr.rawWrite(estr); + } else { + stderr.writeln(t.toString); + } + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(closeConnection) { + ir.source.close(); + break; + } else { + if(!ir.empty) + ir.popFront(); // get the next + else if(ir.sourceClosed) { + ir.source.close(); + } + } + } + + ir.source.close(); + } catch(Throwable t) { + version(CRuntime_Musl) {} else + debug writeln(t); + // most likely cause is a timeout + } + } + } else if(newPid < 0) { + throw new Exception("fork failed"); + } else { + processCount++; + } + } + + // the parent should wait for its children... + if(newPid) { + import core.sys.posix.sys.wait; + + version(embedded_httpd_processes_accept_after_fork) {} else { + import core.sys.posix.sys.select; + int[] fdQueue; + while(true) { + // writeln("select call"); + int nfds = pipeWriteFd; + if(sock > pipeWriteFd) + nfds = sock; + nfds += 1; + fd_set read_fds; + fd_set write_fds; + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_SET(sock, &read_fds); + if(fdQueue.length) + FD_SET(pipeWriteFd, &write_fds); + auto ret = select(nfds, &read_fds, &write_fds, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + goto try_wait; + else + throw new Exception("wtf select"); + } + + int s = -1; + if(FD_ISSET(sock, &read_fds)) { + uint i; + sockaddr addr; + i = addr.sizeof; + s = accept(sock, &addr, &i); + cloexec(s); + import core.sys.posix.netinet.tcp; + int opt = 1; + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } + + if(FD_ISSET(pipeWriteFd, &write_fds)) { + if(s == -1 && fdQueue.length) { + s = fdQueue[0]; + fdQueue = fdQueue[1 .. $]; // FIXME reuse buffer + } + write_fd(pipeWriteFd, &s, s.sizeof, s); + close(s); // we are done with it, let the other process take ownership + } else + fdQueue ~= s; + } + } + + try_wait: + + int status; + while(-1 != wait(&status)) { + version(CRuntime_Musl) {} else { + import std.stdio; writeln("Process died ", status); + } + processCount--; + goto reopen; + } + close(sock); + } +} + +version(fastcgi) +void serveFastCgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(RequestServer params) { + // SetHandler fcgid-script + FCGX_Stream* input, output, error; + FCGX_ParamArray env; + + + + const(ubyte)[] getFcgiChunk() { + const(ubyte)[] ret; + while(FCGX_HasSeenEOF(input) != -1) + ret ~= cast(ubyte) FCGX_GetChar(input); + return ret; + } + + void writeFcgi(const(ubyte)[] data) { + FCGX_PutStr(data.ptr, data.length, output); + } + + void doARequest() { + string[string] fcgienv; + + for(auto e = env; e !is null && *e !is null; e++) { + string cur = to!string(*e); + auto idx = cur.indexOf("="); + string name, value; + if(idx == -1) + name = cur; + else { + name = cur[0 .. idx]; + value = cur[idx + 1 .. $]; + } + + fcgienv[name] = value; + } + + void flushFcgi() { + FCGX_FFlush(output); + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); + } catch(Throwable t) { + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); + return; //continue; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + try { + fun(cgi); + cgi.close(); + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } catch(Throwable t) { + // log it to the error stream + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + // handle it for the user, if we can + if(!handleException(cgi, t)) + return; // continue; + } + } + + auto lp = params.listeningPort; + auto host = params.listeningHost; + + FCGX_Request request; + if(lp || !host.empty) { + // if a listening port was specified on the command line, we want to spawn ourself + // (needed for nginx without spawn-fcgi, e.g. on Windows) + FCGX_Init(); + + int sock; + + if(host.startsWith("unix:")) { + sock = FCGX_OpenSocket(toStringz(params.listeningHost["unix:".length .. $]), 12); + } else if(host.startsWith("abstract:")) { + sock = FCGX_OpenSocket(toStringz("\0" ~ params.listeningHost["abstract:".length .. $]), 12); + } else { + sock = FCGX_OpenSocket(toStringz(params.listeningHost ~ ":" ~ to!string(lp)), 12); + } + + if(sock < 0) + throw new Exception("Couldn't listen on the port"); + FCGX_InitRequest(&request, sock, 0); + while(FCGX_Accept_r(&request) >= 0) { + input = request.inStream; + output = request.outStream; + error = request.errStream; + env = request.envp; + doARequest(); + } + } else { + // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) + // using the version with a global variable since we are separate processes anyway + while(FCGX_Accept(&input, &output, &error, &env) >= 0) { + doARequest(); + } + } +} + +/// Returns the default listening port for the current cgi configuration. 8085 for embedded httpd, 4000 for scgi, irrelevant for others. +ushort defaultListeningPort() { + version(netman_httpd) + return 8080; + else version(embedded_httpd_processes) + return 8085; + else version(embedded_httpd_threads) + return 8085; + else version(scgi) + return 4000; + else + return 0; +} + +/// Default host for listening. 127.0.0.1 for scgi, null (aka all interfaces) for all others. If you want the server directly accessible from other computers on the network, normally use null. If not, 127.0.0.1 is a bit better. Settable with default handlers with --listening-host command line argument. +string defaultListeningHost() { + version(netman_httpd) + return null; + else version(embedded_httpd_processes) + return null; + else version(embedded_httpd_threads) + return null; + else version(scgi) + return "127.0.0.1"; + else + return null; + +} + +/++ + This is the function [GenericMain] calls. View its source for some simple boilerplate you can copy/paste and modify, or you can call it yourself from your `main`. + + Please note that this may spawn other helper processes that will call `main` again. It does this currently for the timer server and event source server (and the quasi-deprecated web socket server). + + Params: + fun = Your request handler + CustomCgi = a subclass of Cgi, if you wise to customize it further + maxContentLength = max POST size you want to allow + args = command-line arguments + + History: + Documented Sept 26, 2020. ++/ +void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) { + if(tryAddonServers(args)) + return; + + if(trySimulatedRequest!(fun, CustomCgi)(args)) + return; + + RequestServer server; + // you can change the port here if you like + // server.listeningPort = 9000; + + // then call this to let the command line args override your default + server.configureFromCommandLine(args); + + // and serve the request(s). + server.serve!(fun, CustomCgi, maxContentLength)(); +} + +//version(plain_cgi) +void handleCgiRequest(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { + // standard CGI is the default version + + + // Set stdin to binary mode if necessary to avoid mangled newlines + // the fact that stdin is global means this could be trouble but standard cgi request + // handling is one per process anyway so it shouldn't actually be threaded here or anything. + version(Windows) { + version(Win64) + _setmode(std.stdio.stdin.fileno(), 0x8000); + else + setmode(std.stdio.stdin.fileno(), 0x8000); + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength); + version(Posix) + cgi._outputFileHandle = cast(CgiConnectionHandle) 1; // stdout + else version(Windows) + cgi._outputFileHandle = cast(CgiConnectionHandle) GetStdHandle(STD_OUTPUT_HANDLE); + else static assert(0); + } catch(Throwable t) { + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto s = t.toString(); + stderr.rawWrite(s); + stdout.rawWrite(plainHttpError(true, "400 Bad Request", t)); + } else { + stderr.writeln(t.msg); + // the real http server will probably handle this; + // most likely, this is a bug in Cgi. But, oh well. + stdout.write(plainHttpError(true, "400 Bad Request", t)); + } + return; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } catch (Throwable t) { + version(CRuntime_Musl) { + // LockingTextWriter fails here + // so working around it + auto s = t.msg; + stderr.rawWrite(s); + } else { + stderr.writeln(t.msg); + } + if(!handleException(cgi, t)) + return; + } +} + +private __gshared int cancelfd = -1; + +/+ + The event loop for embedded_httpd_threads will prolly fiber dispatch + cgi constructors too, so slow posts will not monopolize a worker thread. + + May want to provide the worker task system just need to ensure all the fibers + has a big enough stack for real work... would also ideally like to reuse them. + + + So prolly bir would switch it to nonblocking. If it would block, it epoll + registers one shot with this existing fiber to take it over. + + new connection comes in. it picks a fiber off the free list, + or if there is none, it creates a new one. this fiber handles + this connection the whole time. + + epoll triggers the fiber when something comes in. it is called by + a random worker thread, it might change at any time. at least during + the constructor. maybe into the main body it will stay tied to a thread + just so TLS stuff doesn't randomly change in the middle. but I could + specify if you yield all bets are off. + + when the request is finished, if there's more data buffered, it just + keeps going. if there is no more data buffered, it epoll ctls to + get triggered when more data comes in. all one shot. + + when a connection is closed, the fiber returns and is then reset + and added to the free list. if the free list is full, the fiber is + just freed, this means it will balloon to a certain size but not generally + grow beyond that unless the activity keeps going. + + 256 KB stack i thnk per fiber. 4,000 active fibers per gigabyte of memory. + + So the fiber has its own magic methods to read and write. if they would block, it registers + for epoll and yields. when it returns, it read/writes and then returns back normal control. + + basically you issue the command and it tells you when it is done + + it needs to DEL the epoll thing when it is closed. add it when opened. mod it when anther thing issued + ++/ + +/++ + The stack size when a fiber is created. You can set this from your main or from a shared static constructor + to optimize your memory use if you know you don't need this much space. Be careful though, some functions use + more stack space than you realize and a recursive function (including ones like in dom.d) can easily grow fast! + + History: + Added July 10, 2021. Previously, it used the druntime default of 16 KB. ++/ +version(cgi_use_fiber) +__gshared size_t fiberStackSize = 4096 * 100; + +version(cgi_use_fiber) +class CgiFiber : Fiber { + private void function(Socket) f_handler; + private void f_handler_dg(Socket s) { // to avoid extra allocation w/ function + f_handler(s); + } + this(void function(Socket) handler) { + this.f_handler = handler; + this(&f_handler_dg); + } + + this(void delegate(Socket) handler) { + this.handler = handler; + super(&run, fiberStackSize); + } + + Socket connection; + void delegate(Socket) handler; + + void run() { + handler(connection); + } + + void delegate() postYield; + + private void setPostYield(scope void delegate() py) @nogc { + postYield = cast(void delegate()) py; + } + + void proceed() { + try { + call(); + auto py = postYield; + postYield = null; + if(py !is null) + py(); + } catch(Exception e) { + if(connection) + connection.close(); + goto terminate; + } + + if(state == State.TERM) { + terminate: + import core.memory; + GC.removeRoot(cast(void*) this); + } + } +} + +version(cgi_use_fiber) +version(Windows) { + +extern(Windows) private { + + import core.sys.windows.mswsock; + + alias GROUP=uint; + alias LPWSAPROTOCOL_INFOW = void*; + SOCKET WSASocketW(int af, int type, int protocol, LPWSAPROTOCOL_INFOW lpProtocolInfo, GROUP g, DWORD dwFlags); + alias WSASend = arsd.core.WSASend; + alias WSARecv = arsd.core.WSARecv; + alias WSABUF = arsd.core.WSABUF; + + /+ + int WSASend(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesSent, DWORD dwFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); + int WSARecv(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesRecvd, LPDWORD lpFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); + + struct WSABUF { + ULONG len; + CHAR *buf; + } + +/ + alias LPWSABUF = WSABUF*; + + alias WSAOVERLAPPED = OVERLAPPED; + alias LPWSAOVERLAPPED = LPOVERLAPPED; + /+ + + alias LPFN_ACCEPTEX = + BOOL + function( + SOCKET sListenSocket, + SOCKET sAcceptSocket, + //_Out_writes_bytes_(dwReceiveDataLength+dwLocalAddressLength+dwRemoteAddressLength) PVOID lpOutputBuffer, + void* lpOutputBuffer, + WORD dwReceiveDataLength, + WORD dwLocalAddressLength, + WORD dwRemoteAddressLength, + LPDWORD lpdwBytesReceived, + LPOVERLAPPED lpOverlapped + ); + + enum WSAID_ACCEPTEX = GUID([0xb5367df1,0xcbac,0x11cf,[0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92]]); + +/ + + enum WSAID_GETACCEPTEXSOCKADDRS = GUID(0xb5367df2,0xcbac,0x11cf,[0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92]); +} + +private class PseudoblockingOverlappedSocket : Socket { + SOCKET handle; + + CgiFiber fiber; + + this(AddressFamily af, SocketType st) { + auto handle = WSASocketW(af, st, 0, null, 0, 1 /*WSA_FLAG_OVERLAPPED*/); + if(!handle) + throw new Exception("WSASocketW"); + this.handle = handle; + + iocp = CreateIoCompletionPort(cast(HANDLE) handle, iocp, cast(ULONG_PTR) cast(void*) this, 0); + + if(iocp is null) { + writeln(GetLastError()); + throw new Exception("CreateIoCompletionPort"); + } + + super(cast(socket_t) handle, af); + } + this() pure nothrow @trusted { assert(0); } + + override void blocking(bool) {} // meaningless to us, just ignore it. + + protected override Socket accepting() pure nothrow { + assert(0); + } + + bool addressesParsed; + Address la; + Address ra; + + private void populateAddresses() { + if(addressesParsed) + return; + addressesParsed = true; + + int lalen, ralen; + + sockaddr_in* la; + sockaddr_in* ra; + + lpfnGetAcceptExSockaddrs( + scratchBuffer.ptr, + 0, // same as in the AcceptEx call! + sockaddr_in.sizeof + 16, + sockaddr_in.sizeof + 16, + cast(sockaddr**) &la, + &lalen, + cast(sockaddr**) &ra, + &ralen + ); + + if(la) + this.la = new InternetAddress(*la); + if(ra) + this.ra = new InternetAddress(*ra); + + } + + override @property @trusted Address localAddress() { + populateAddresses(); + return la; + } + override @property @trusted Address remoteAddress() { + populateAddresses(); + return ra; + } + + PseudoblockingOverlappedSocket accepted; + + __gshared static LPFN_ACCEPTEX lpfnAcceptEx; + __gshared static typeof(&GetAcceptExSockaddrs) lpfnGetAcceptExSockaddrs; + + override Socket accept() @trusted { + __gshared static LPFN_ACCEPTEX lpfnAcceptEx; + + if(lpfnAcceptEx is null) { + DWORD dwBytes; + GUID GuidAcceptEx = WSAID_ACCEPTEX; + + auto iResult = WSAIoctl(handle, 0xc8000006 /*SIO_GET_EXTENSION_FUNCTION_POINTER*/, + &GuidAcceptEx, GuidAcceptEx.sizeof, + &lpfnAcceptEx, lpfnAcceptEx.sizeof, + &dwBytes, null, null); + + GuidAcceptEx = WSAID_GETACCEPTEXSOCKADDRS; + iResult = WSAIoctl(handle, 0xc8000006 /*SIO_GET_EXTENSION_FUNCTION_POINTER*/, + &GuidAcceptEx, GuidAcceptEx.sizeof, + &lpfnGetAcceptExSockaddrs, lpfnGetAcceptExSockaddrs.sizeof, + &dwBytes, null, null); + + } + + auto pfa = new PseudoblockingOverlappedSocket(AddressFamily.INET, SocketType.STREAM); + accepted = pfa; + + SOCKET pendingForAccept = pfa.handle; + DWORD ignored; + + auto ret = lpfnAcceptEx(handle, + pendingForAccept, + // buffer to receive up front + pfa.scratchBuffer.ptr, + 0, + // size of local and remote addresses. normally + 16. + sockaddr_in.sizeof + 16, + sockaddr_in.sizeof + 16, + &ignored, // bytes would be given through the iocp instead but im not even requesting the thing + &overlapped + ); + + return pfa; + } + + override void connect(Address to) { assert(0); } + + DWORD lastAnswer; + ubyte[1024] scratchBuffer; + static assert(scratchBuffer.length > sockaddr_in.sizeof * 2 + 32); + + WSABUF[1] buffer; + OVERLAPPED overlapped; + override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) @trusted { + overlapped = overlapped.init; + buffer[0].len = cast(DWORD) buf.length; + buffer[0].buf = cast(ubyte*) buf.ptr; + fiber.setPostYield( () { + if(!WSASend(handle, buffer.ptr, cast(DWORD) buffer.length, null, 0, &overlapped, null)) { + if(GetLastError() != 997) { + //throw new Exception("WSASend fail"); + } + } + }); + + Fiber.yield(); + return lastAnswer; + } + override ptrdiff_t receive(scope void[] buf, SocketFlags flags) @trusted { + overlapped = overlapped.init; + buffer[0].len = cast(DWORD) buf.length; + buffer[0].buf = cast(ubyte*) buf.ptr; + + DWORD flags2 = 0; + + fiber.setPostYield(() { + if(!WSARecv(handle, buffer.ptr, cast(DWORD) buffer.length, null, &flags2 /* flags */, &overlapped, null)) { + if(GetLastError() != 997) { + //writeln("WSARecv ", WSAGetLastError()); + //throw new Exception("WSARecv fail"); + } + } + }); + + Fiber.yield(); + return lastAnswer; + } + + // I might go back and implement these for udp things. + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags, ref Address from) @trusted { + assert(0); + } + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags) @trusted { + assert(0); + } + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags, Address to) @trusted { + assert(0); + } + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags) @trusted { + assert(0); + } + + // lol overload sets + alias send = typeof(super).send; + alias receive = typeof(super).receive; + alias sendTo = typeof(super).sendTo; + alias receiveFrom = typeof(super).receiveFrom; + +} +} + +void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { + assert(connection !is null); + version(cgi_use_fiber) { + auto fiber = new CgiFiber(&doThreadHttpConnectionGuts!(CustomCgi, fun)); + + version(Windows) { + (cast(PseudoblockingOverlappedSocket) connection).fiber = fiber; + } + + import core.memory; + GC.addRoot(cast(void*) fiber); + fiber.connection = connection; + fiber.proceed(); + } else { + doThreadHttpConnectionGuts!(CustomCgi, fun)(connection); + } +} + +/+ + +/+ + The represents a recyclable per-task arena allocator. The default is to let the GC manage the whole block as a large array, meaning if a reference into it is escaped, it waste memory but is not dangerous. If you don't escape any references to it and don't do anything special, the GC collects it. + + But, if you call `cgi.recyclable = true`, the memory is retained for the next request on the thread. If a reference is escaped, it is the user's problem; it can be modified (and break the `immutable` guarantees!) and thus be memory unsafe. They're taking responsibility for doing it right when they call `escape`. But if they do it right and opt into recycling, the memory is all reused to give a potential boost without requiring the GC's involvement. + + What if one request used an abnormally large amount of memory though? Will recycling it keep that pinned forever? No, that's why it keeps track of some stats. If a working set was significantly above average and not fully utilized for a while, it will just let the GC have it again despite your suggestion to recycle it. + + Be warned that growing the memory block may release the old, smaller block for garbage collection. If you retained references to it, it may not be collectable and lead to some unnecessary memory growth. It is probably best to try to keep the things sized in a continuous block that doesn't have to grow often. + + Internally, it is broken up into a few blocks: + * the request block. This holds the incoming request and associated data (parsed headers, variables, etc). + * the scannable block. this holds pointers arrays, classes, etc. associated with this request, so named because the GC scans it. + * the response block. This holds the output buffer. + + And I may add more later if I decide to open this up to outside user code. + + The scannable block is separate to limit the amount of work the GC has to do; no point asking it to scan that which need not be scanned. + + The request and response blocks are separated because they will have different typical sizes, with the request likely being less predictable. Being able to release one to the GC while recycling the other might help, and having them grow independently (if needed) may also prevent some pain. + + All of this are internal implementation details subject to change at any time without notice. It is valid for my recycle method to do absolutely nothing; the GC also eventually recycles memory! + + Each active task can have its own recyclable memory object. When you recycle it, it is added to a thread-local freelist. If the list is excessively large, entries maybe discarded at random and left for the GC to prevent a temporary burst of activity from leading to a permanent waste of memory. ++/ +struct RecyclableMemory { + private ubyte[] inputBuffer; + private ubyte[] processedRequestBlock; + private void[] scannableBlock; + private ubyte[] outputBuffer; + + RecyclableMemory* next; +} + +/++ + This emulates the D associative array interface with a different internal implementation. + + string s = cgi.get["foo"]; // just does cgi.getArray[x][$-1]; + string[] arr = cgi.getArray["foo"]; + + "foo" in cgi.get + + foreach(k, v; cgi.get) + + cgi.get.toAA // for compatibility + + // and this can urldecode lazily tbh... in-place even, since %xx is always longer than a single char thing it turns into... + ... but how does it mark that it has already been processed in-place? it'd have to just add it to the index then. + + deprecated alias toAA this; ++/ +struct VariableCollection { + private VariableArrayCollection* vac; + + const(char[]) opIndex(scope const char[] key) { + return (*vac)[key][$-1]; + } + + const(char[]*) opBinaryRight(string op : "in")(scope const char[] key) { + return key in (*vac); + } + + int opApply(int delegate(scope const(char)[] key, scope const(char)[] value) dg) { + foreach(k, v; *vac) { + if(auto res = dg(k, v[$-1])) + return res; + } + return 0; + } + + immutable(string[string]) toAA() { + string[string] aa; + foreach(k, v; *vac) + aa[k.idup] = v[$-1].idup; + return aa; + } + + deprecated alias toAA this; +} + +struct VariableArrayCollection { + /+ + This needs the actual implementation of looking it up. As it pulls data, it should + decode and index for later. + + The index will go into a block attached to the cgi object and it should prolly be sorted + something like + + [count of names] + [slice to name][count of values][slice to value, decoded in-place, ...] + ... + +/ + private Cgi cgi; + + const(char[][]) opIndex(scope const char[] key) { + return null; + } + + const(char[][]*) opBinaryRight(string op : "in")(scope const char[] key) { + return null; + } + + // int opApply(int delegate(scope const(char)[] key, scope const(char)[][] value) dg) + + immutable(string[string]) toAA() { + return null; + } + + deprecated alias toAA this; + +} + +struct HeaderCollection { + +} ++/ + +void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection = false)(Socket connection) { + scope(failure) { + // catch all for other errors + try { + sendAll(connection, plainHttpError(false, "500 Internal Server Error", null)); + connection.close(); + } catch(Exception e) {} // swallow it, we're aborting anyway. + } + + bool closeConnection = alwaysCloseConnection; + + /+ + ubyte[4096] inputBuffer = void; + ubyte[__traits(classInstanceSize, BufferedInputRange)] birBuffer = void; + ubyte[__traits(classInstanceSize, CustomCgi)] cgiBuffer = void; + + birBuffer[] = cast(ubyte[]) typeid(BufferedInputRange).initializer()[]; + BufferedInputRange ir = cast(BufferedInputRange) cast(void*) birBuffer.ptr; + ir.__ctor(connection, inputBuffer[], true); + +/ + + auto ir = new BufferedInputRange(connection); + + while(!ir.empty) { + + if(ir.view.length == 0) { + ir.popFront(); + if(ir.sourceClosed) { + connection.close(); + closeConnection = true; + break; + } + } + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + // There's a bunch of these casts around because the type matches up with + // the -version=.... specifiers, just you can also create a RequestServer + // and instantiate the things where the types don't match up. It isn't exactly + // correct but I also don't care rn. Might FIXME and either remove it later or something. + cgi._outputFileHandle = cast(CgiConnectionHandle) connection.handle; + } catch(ConnectionClosedException ce) { + closeConnection = true; + break; + } catch(ConnectionException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + break; + } catch(HttpVersionNotSupportedException ve) { + sendAll(connection, plainHttpError(false, "505 HTTP Version Not Supported", ve)); + closeConnection = true; + break; + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + version(CRuntime_Musl) { + stderr.rawWrite(t.toString()); + stderr.rawWrite("\n"); + } else { + stderr.writeln(t.toString()); + } + sendAll(connection, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + if(cgi.websocketMode) + closeConnection = true; + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } catch(ConnectionException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + } catch(ConnectionClosedException ce) { + // broken pipe or something, just abort the connection + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + version(CRuntime_Musl) {} else + stderr.writeln(t.toString); + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(globalStopFlag) + closeConnection = true; + + if(closeConnection || alwaysCloseConnection) { + connection.shutdown(SocketShutdown.BOTH); + connection.close(); + ir.dispose(); + closeConnection = false; // don't reclose after loop + break; + } else { + if(ir.front.length) { + ir.popFront(); // we can't just discard the buffer, so get the next bit and keep chugging along + } else if(ir.sourceClosed) { + ir.source.shutdown(SocketShutdown.BOTH); + ir.source.close(); + ir.dispose(); + closeConnection = false; + } else { + continue; + // break; // this was for a keepalive experiment + } + } + } + + if(closeConnection) { + connection.shutdown(SocketShutdown.BOTH); + connection.close(); + ir.dispose(); + } + + // I am otherwise NOT closing it here because the parent thread might still be able to make use of the keep-alive connection! +} + +void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket connection) { + // and now we can buffer + scope(failure) + connection.close(); + + import al = std.algorithm; + + size_t size; + + string[string] headers; + + auto range = new BufferedInputRange(connection); + more_data: + auto chunk = range.front(); + // waiting for colon for header length + auto idx = indexOf(cast(string) chunk, ':'); + if(idx == -1) { + try { + range.popFront(); + } catch(Exception e) { + // it is just closed, no big deal + connection.close(); + return; + } + goto more_data; + } + + size = to!size_t(cast(string) chunk[0 .. idx]); + chunk = range.consume(idx + 1); + // reading headers + if(chunk.length < size) + range.popFront(0, size + 1); + // we are now guaranteed to have enough + chunk = range.front(); + assert(chunk.length > size); + + idx = 0; + string key; + string value; + foreach(part; al.splitter(chunk, '\0')) { + if(idx & 1) { // odd is value + value = cast(string)(part.idup); + headers[key] = value; // commit + } else + key = cast(string)(part.idup); + idx++; + } + + enforce(chunk[size] == ','); // the terminator + + range.consume(size + 1); + // reading data + // this will be done by Cgi + + const(ubyte)[] getScgiChunk() { + // we are already primed + auto data = range.front(); + if(data.length == 0 && !range.sourceClosed) { + range.popFront(0); + data = range.front(); + } else if (range.sourceClosed) + range.source.close(); + + return data; + } + + void writeScgi(const(ubyte)[] data) { + sendAll(connection, data); + } + + void flushScgi() { + // I don't *think* I have to do anything.... + } + + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength, headers, &getScgiChunk, &writeScgi, &flushScgi); + cgi._outputFileHandle = cast(CgiConnectionHandle) connection.handle; + } catch(Throwable t) { + sendAll(connection, plainHttpError(true, "400 Bad Request", t)); + connection.close(); + return; // this connection is dead + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + try { + fun(cgi); + cgi.close(); + connection.close(); + + } catch(AuthorizationRequiredException are) { + cgi.setResponseStatus("401 Authorization Required"); + cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\""); + cgi.close(); + } catch(Throwable t) { + // no std err + if(!handleException(cgi, t)) { + connection.close(); + return; + } else { + connection.close(); + return; + } + } +} + +string printDate(DateTime date) { + char[29] buffer = void; + printDateToBuffer(date, buffer[]); + return buffer.idup; +} + +int printDateToBuffer(DateTime date, char[] buffer) @nogc { + assert(buffer.length >= 29); + // 29 static length ? + + static immutable daysOfWeek = [ + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" + ]; + + static immutable months = [ + null, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + ]; + + buffer[0 .. 3] = daysOfWeek[date.dayOfWeek]; + buffer[3 .. 5] = ", "; + buffer[5] = date.day / 10 + '0'; + buffer[6] = date.day % 10 + '0'; + buffer[7] = ' '; + buffer[8 .. 11] = months[date.month]; + buffer[11] = ' '; + auto y = date.year; + buffer[12] = cast(char) (y / 1000 + '0'); y %= 1000; + buffer[13] = cast(char) (y / 100 + '0'); y %= 100; + buffer[14] = cast(char) (y / 10 + '0'); y %= 10; + buffer[15] = cast(char) (y + '0'); + buffer[16] = ' '; + buffer[17] = date.hour / 10 + '0'; + buffer[18] = date.hour % 10 + '0'; + buffer[19] = ':'; + buffer[20] = date.minute / 10 + '0'; + buffer[21] = date.minute % 10 + '0'; + buffer[22] = ':'; + buffer[23] = date.second / 10 + '0'; + buffer[24] = date.second % 10 + '0'; + buffer[25 .. $] = " GMT"; + + return 29; +} + + +// Referencing this gigantic typeid seems to remind the compiler +// to actually put the symbol in the object file. I guess the immutable +// assoc array array isn't actually included in druntime +void hackAroundLinkerError() { + stdout.rawWrite(typeid(const(immutable(char)[][])[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(char)[][][immutable(char)[]]).toString()); + stdout.rawWrite(typeid(Cgi.UploadedFile[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(Cgi.UploadedFile[][immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(Cgi.UploadedFile)[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(Cgi.UploadedFile[])[immutable(char)[]]).toString()); + stdout.rawWrite(typeid(immutable(char[])[immutable(char)[]]).toString()); + // this is getting kinda ridiculous btw. Moving assoc arrays + // to the library is the pain that keeps on coming. + + // eh this broke the build on the work server + // stdout.rawWrite(typeid(immutable(char)[][immutable(string[])])); + stdout.rawWrite(typeid(immutable(string[])[immutable(char)[]]).toString()); +} + + + + + +version(fastcgi) { + pragma(lib, "fcgi"); + + static if(size_t.sizeof == 8) // 64 bit + alias long c_int; + else + alias int c_int; + + extern(C) { + struct FCGX_Stream { + ubyte* rdNext; + ubyte* wrNext; + ubyte* stop; + ubyte* stopUnget; + c_int isReader; + c_int isClosed; + c_int wasFCloseCalled; + c_int FCGI_errno; + void* function(FCGX_Stream* stream) fillBuffProc; + void* function(FCGX_Stream* stream, c_int doClose) emptyBuffProc; + void* data; + } + + // note: this is meant to be opaque, so don't access it directly + struct FCGX_Request { + int requestId; + int role; + FCGX_Stream* inStream; + FCGX_Stream* outStream; + FCGX_Stream* errStream; + char** envp; + void* paramsPtr; + int ipcFd; + int isBeginProcessed; + int keepConnection; + int appStatus; + int nWriters; + int flags; + int listen_sock; + } + + int FCGX_InitRequest(FCGX_Request *request, int sock, int flags); + void FCGX_Init(); + + int FCGX_Accept_r(FCGX_Request *request); + + + alias char** FCGX_ParamArray; + + c_int FCGX_Accept(FCGX_Stream** stdin, FCGX_Stream** stdout, FCGX_Stream** stderr, FCGX_ParamArray* envp); + c_int FCGX_GetChar(FCGX_Stream* stream); + c_int FCGX_PutStr(const ubyte* str, c_int n, FCGX_Stream* stream); + int FCGX_HasSeenEOF(FCGX_Stream* stream); + c_int FCGX_FFlush(FCGX_Stream *stream); + + int FCGX_OpenSocket(in char*, int); + } +} + + +/* This might go int a separate module eventually. It is a network input helper class. */ + +import std.socket; + +version(cgi_use_fiber) { + import core.thread; + + version(linux) { + import core.sys.linux.epoll; + + int epfd = -1; // thread local because EPOLLEXCLUSIVE works much better this way... weirdly. + } else version(Windows) { + // declaring the iocp thing below... + } else static assert(0, "The hybrid fiber server is not implemented on your OS."); +} + +version(Windows) + __gshared HANDLE iocp; + +version(cgi_use_fiber) { + version(linux) + private enum WakeupEvent { + Read = EPOLLIN, + Write = EPOLLOUT + } + else version(Windows) + private enum WakeupEvent { + Read, Write + } + else static assert(0); +} + +version(cgi_use_fiber) +private void registerEventWakeup(bool* registered, Socket source, WakeupEvent e) @nogc { + + // static cast since I know what i have in here and don't want to pay for dynamic cast + auto f = cast(CgiFiber) cast(void*) Fiber.getThis(); + + version(linux) { + f.setPostYield = () { + if(*registered) { + // rearm + epoll_event evt; + evt.events = e | EPOLLONESHOT; + evt.data.ptr = cast(void*) f; + if(epoll_ctl(epfd, EPOLL_CTL_MOD, source.handle, &evt) == -1) + throw new Exception("epoll_ctl"); + } else { + // initial registration + *registered = true ; + int fd = source.handle; + epoll_event evt; + evt.events = e | EPOLLONESHOT; + evt.data.ptr = cast(void*) f; + if(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &evt) == -1) + throw new Exception("epoll_ctl"); + } + }; + + Fiber.yield(); + + f.setPostYield(null); + } else version(Windows) { + Fiber.yield(); + } + else static assert(0); +} + +version(cgi_use_fiber) +void unregisterSource(Socket s) { + version(linux) { + epoll_event evt; + epoll_ctl(epfd, EPOLL_CTL_DEL, s.handle(), &evt); + } else version(Windows) { + // intentionally blank + } + else static assert(0); +} + +// it is a class primarily for reference semantics +// I might change this interface +/// This is NOT ACTUALLY an input range! It is too different. Historical mistake kinda. +class BufferedInputRange { + version(Posix) + this(int source, ubyte[] buffer = null) { + this(new Socket(cast(socket_t) source, AddressFamily.INET), buffer); + } + + this(Socket source, ubyte[] buffer = null, bool allowGrowth = true) { + // if they connect but never send stuff to us, we don't want it wasting the process + // so setting a time out + version(cgi_use_fiber) + source.blocking = false; + else + source.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(3)); + + this.source = source; + if(buffer is null) { + underlyingBuffer = new ubyte[4096]; + this.allowGrowth = true; + } else { + underlyingBuffer = buffer; + this.allowGrowth = allowGrowth; + } + + assert(underlyingBuffer.length); + + // we assume view.ptr is always inside underlyingBuffer + view = underlyingBuffer[0 .. 0]; + + popFront(); // prime + } + + version(cgi_use_fiber) { + bool registered; + } + + void dispose() { + version(cgi_use_fiber) { + if(registered) + unregisterSource(source); + } + } + + /** + A slight difference from regular ranges is you can give it the maximum + number of bytes to consume. + + IMPORTANT NOTE: the default is to consume nothing, so if you don't call + consume() yourself and use a regular foreach, it will infinitely loop! + + The default is to do what a normal range does, and consume the whole buffer + and wait for additional input. + + You can also specify 0, to append to the buffer, or any other number + to remove the front n bytes and wait for more. + */ + void popFront(size_t maxBytesToConsume = 0 /*size_t.max*/, size_t minBytesToSettleFor = 0, bool skipConsume = false) { + if(sourceClosed) + throw new ConnectionClosedException("can't get any more data from a closed source"); + if(!skipConsume) + consume(maxBytesToConsume); + + // we might have to grow the buffer + if(minBytesToSettleFor > underlyingBuffer.length || view.length == underlyingBuffer.length) { + if(allowGrowth) { + //import std.stdio; writeln("growth"); + auto viewStart = view.ptr - underlyingBuffer.ptr; + size_t growth = 4096; + // make sure we have enough for what we're being asked for + if(minBytesToSettleFor > 0 && minBytesToSettleFor - underlyingBuffer.length > growth) + growth = minBytesToSettleFor - underlyingBuffer.length; + //import std.stdio; writeln(underlyingBuffer.length, " ", viewStart, " ", view.length, " ", growth, " ", minBytesToSettleFor, " ", minBytesToSettleFor - underlyingBuffer.length); + underlyingBuffer.length += growth; + view = underlyingBuffer[viewStart .. view.length]; + } else + throw new Exception("No room left in the buffer"); + } + + do { + auto freeSpace = underlyingBuffer[view.ptr - underlyingBuffer.ptr + view.length .. $]; + try_again: + auto ret = source.receive(freeSpace); + if(ret == Socket.ERROR) { + if(wouldHaveBlocked()) { + version(cgi_use_fiber) { + registerEventWakeup(®istered, source, WakeupEvent.Read); + goto try_again; + } else { + // gonna treat a timeout here as a close + sourceClosed = true; + return; + } + } + version(Posix) { + import core.stdc.errno; + if(errno == EINTR || errno == EAGAIN) { + goto try_again; + } + if(errno == ECONNRESET) { + sourceClosed = true; + return; + } + } + throw new Exception(lastSocketError); // FIXME + } + if(ret == 0) { + sourceClosed = true; + return; + } + + //import std.stdio; writeln(view.ptr); writeln(underlyingBuffer.ptr); writeln(view.length, " ", ret, " = ", view.length + ret); + view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; + //import std.stdio; writeln(cast(string) view); + } while(view.length < minBytesToSettleFor); + } + + /// Removes n bytes from the front of the buffer, and returns the new buffer slice. + /// You might want to idup the data you are consuming if you store it, since it may + /// be overwritten on the new popFront. + /// + /// You do not need to call this if you always want to wait for more data when you + /// consume some. + ubyte[] consume(size_t bytes) { + //import std.stdio; writeln("consuime ", bytes, "/", view.length); + view = view[bytes > $ ? $ : bytes .. $]; + if(view.length == 0) { + view = underlyingBuffer[0 .. 0]; // go ahead and reuse the beginning + /* + writeln("HERE"); + popFront(0, 0, true); // try to load more if we can, checks if the source is closed + writeln(cast(string)front); + writeln("DONE"); + */ + } + return front; + } + + bool empty() { + return sourceClosed && view.length == 0; + } + + ubyte[] front() { + return view; + } + + @system invariant() { + assert(view.ptr >= underlyingBuffer.ptr); + // it should never be equal, since if that happens view ought to be empty, and thus reusing the buffer + assert(view.ptr < underlyingBuffer.ptr + underlyingBuffer.length); + } + + ubyte[] underlyingBuffer; + bool allowGrowth; + ubyte[] view; + Socket source; + bool sourceClosed; +} + +private class FakeSocketForStdin : Socket { + import std.stdio; + + this() { + + } + + private bool closed; + + override ptrdiff_t receive(scope void[] buffer, std.socket.SocketFlags) @trusted { + if(closed) + throw new Exception("Closed"); + return stdin.rawRead(buffer).length; + } + + override ptrdiff_t send(const scope void[] buffer, std.socket.SocketFlags) @trusted { + if(closed) + throw new Exception("Closed"); + stdout.rawWrite(buffer); + return buffer.length; + } + + override void close() @trusted scope { + (cast(void delegate() @nogc nothrow) &realClose)(); + } + + override void shutdown(SocketShutdown s) { + // FIXME + } + + override void setOption(SocketOptionLevel, SocketOption, scope void[]) {} + override void setOption(SocketOptionLevel, SocketOption, Duration) {} + + override @property @trusted Address remoteAddress() { return null; } + override @property @trusted Address localAddress() { return null; } + + void realClose() { + closed = true; + try { + stdin.close(); + stdout.close(); + } catch(Exception e) { + + } + } +} + +import core.sync.semaphore; +import core.atomic; + +/** + To use this thing: + + --- + void handler(Socket s) { do something... } + auto manager = new ListeningConnectionManager("127.0.0.1", 80, &handler, &delegateThatDropsPrivileges); + manager.listen(); + --- + + The 4th parameter is optional. + + I suggest you use BufferedInputRange(connection) to handle the input. As a packet + comes in, you will get control. You can just continue; though to fetch more. + + + FIXME: should I offer an event based async thing like netman did too? Yeah, probably. +*/ +class ListeningConnectionManager { + Semaphore semaphore; + Socket[256] queue; + shared(ubyte) nextIndexFront; + ubyte nextIndexBack; + shared(int) queueLength; + + Socket acceptCancelable() { + version(Posix) { + import core.sys.posix.sys.select; + fd_set read_fds; + FD_ZERO(&read_fds); + int max = 0; + foreach(listener; listeners) { + FD_SET(listener.handle, &read_fds); + if(listener.handle > max) + max = listener.handle; + } + if(cancelfd != -1) { + FD_SET(cancelfd, &read_fds); + if(cancelfd > max) + max = cancelfd; + } + auto ret = select(max + 1, &read_fds, null, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + return null; + else + throw new Exception("wtf select"); + } + + if(cancelfd != -1 && FD_ISSET(cancelfd, &read_fds)) { + return null; + } + + foreach(listener; listeners) { + if(FD_ISSET(listener.handle, &read_fds)) + return listener.accept(); + } + + return null; + } else { + + auto check = new SocketSet(); + + keep_looping: + check.reset(); + foreach(listener; listeners) + check.add(listener); + + // just to check the stop flag on a kinda busy loop. i hate this FIXME + auto got = Socket.select(check, null, null, 3.seconds); + if(got > 0) + foreach(listener; listeners) + if(check.isSet(listener)) + return listener.accept(); + if(globalStopFlag) + return null; + else + goto keep_looping; + } + } + + int defaultNumberOfThreads() { + import std.parallelism; + version(cgi_use_fiber) { + return totalCPUs * 2 + 1; // still chance some will be pointlessly blocked anyway + } else { + // I times 4 here because there's a good chance some will be blocked on i/o. + return totalCPUs * 4; + } + + } + + void listen() { + shared(int) loopBroken; + + version(Posix) { + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + } + + version(linux) { + if(cancelfd == -1) + cancelfd = eventfd(0, 0); + } + + version(cgi_no_threads) { + // NEVER USE THIS + // it exists only for debugging and other special occasions + + // the thread mode is faster and less likely to stall the whole + // thing when a request is slow + while(!loopBroken && !globalStopFlag) { + auto sn = acceptCancelable(); + if(sn is null) continue; + cloexec(sn); + try { + handler(sn); + } catch(Exception e) { + // if a connection goes wrong, we want to just say no, but try to carry on unless it is an Error of some sort (in which case, we'll die. You might want an external helper program to revive the server when it dies) + sn.close(); + } + } + } else { + + if(useFork) { + version(linux) { + //asm { int 3; } + fork(); + } + } + + version(cgi_use_fiber) { + + version(Windows) { + // please note these are overlapped sockets! so the accept just kicks things off + foreach(listener; listeners) + listener.accept(); + } + + WorkerThread[] threads = new WorkerThread[](numberOfThreads); + foreach(i, ref thread; threads) { + thread = new WorkerThread(this, handler, cast(int) i); + thread.start(); + } + + bool fiber_crash_check() { + bool hasAnyRunning; + foreach(thread; threads) { + if(!thread.isRunning) { + thread.join(); + } else hasAnyRunning = true; + } + + return (!hasAnyRunning); + } + + + while(!globalStopFlag) { + Thread.sleep(1.seconds); + if(fiber_crash_check()) + break; + } + + } else { + semaphore = new Semaphore(); + + ConnectionThread[] threads = new ConnectionThread[](numberOfThreads); + foreach(i, ref thread; threads) { + thread = new ConnectionThread(this, handler, cast(int) i); + thread.start(); + } + + while(!loopBroken && !globalStopFlag) { + Socket sn; + + bool crash_check() { + bool hasAnyRunning; + foreach(thread; threads) { + if(!thread.isRunning) { + thread.join(); + } else hasAnyRunning = true; + } + + return (!hasAnyRunning); + } + + + void accept_new_connection() { + sn = acceptCancelable(); + if(sn is null) return; + cloexec(sn); + if(tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + } + + void existing_connection_new_data() { + // wait until a slot opens up + // int waited = 0; + while(queueLength >= queue.length) { + Thread.sleep(1.msecs); + // waited ++; + } + // if(waited) {import std.stdio; writeln(waited);} + synchronized(this) { + queue[nextIndexBack] = sn; + nextIndexBack++; + atomicOp!"+="(queueLength, 1); + } + semaphore.notify(); + } + + + accept_new_connection(); + if(sn !is null) + existing_connection_new_data(); + else if(sn is null && globalStopFlag) { + foreach(thread; threads) { + semaphore.notify(); + } + Thread.sleep(50.msecs); + } + + if(crash_check()) + break; + } + } + + // FIXME: i typically stop this with ctrl+c which never + // actually gets here. i need to do a sigint handler. + if(cleanup) + cleanup(); + } + } + + //version(linux) + //int epoll_fd; + + bool tcp; + void delegate() cleanup; + + private void function(Socket) fhandler; + private void dg_handler(Socket s) { + fhandler(s); + } + + + this(string[] listenSpec, void function(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + fhandler = handler; + this(listenSpec, &dg_handler, dropPrivs, useFork, numberOfThreads); + } + this(string[] listenSpec, void delegate(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + string[] host; + ushort[] port; + + foreach(spec; listenSpec) { + /+ + The format: + + protocol:// + address_spec + + Protocol is optional. Must be http, https, scgi, or fastcgi. + + address_spec is either: + ipv4 address : port + [ipv6 address] : port + unix:filename + abstract:name + port + +/ + + string protocol; + string address_spec; + + auto protocolIdx = spec.indexOf("://"); + if(protocolIdx != -1) { + protocol = spec[0 .. protocolIdx]; + address_spec = spec[protocolIdx + "://".length .. $]; + } else { + address_spec = spec; + } + + if(address_spec.startsWith("unix:") || address_spec.startsWith("abstract:")) { + host ~= address_spec; + port ~= 0; + } else { + auto idx = address_spec.lastIndexOf(":"); + if(idx == -1) { + host ~= null; + } else { + auto as = address_spec[0 .. idx]; + if(as.length >= 3 && as[0] == '[' && as[$-1] == ']') + as = as[1 .. $-1]; + host ~= as; + } + port ~= address_spec[idx + 1 .. $].to!ushort; + } + + } + + this(host, port, handler, dropPrivs, useFork, numberOfThreads); + } + + this(string host, ushort port, void function(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + this([host], [port], handler, dropPrivs, useFork, numberOfThreads); + } + this(string host, ushort port, void delegate(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + this([host], [port], handler, dropPrivs, useFork, numberOfThreads); + } + + this(string[] host, ushort[] port, void function(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + fhandler = handler; + this(host, port, &dg_handler, dropPrivs, useFork, numberOfThreads); + } + + this(string[] host, ushort[] port, void delegate(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { + assert(host.length == port.length); + + this.handler = handler; + this.useFork = useFork; + this.numberOfThreads = numberOfThreads ? numberOfThreads : defaultNumberOfThreads(); + + listeners.reserve(host.length); + + foreach(i; 0 .. host.length) + if(host[i] == "localhost") { + listeners ~= startListening("127.0.0.1", port[i], tcp, cleanup, 128, dropPrivs); + listeners ~= startListening("::1", port[i], tcp, cleanup, 128, dropPrivs); + } else { + listeners ~= startListening(host[i], port[i], tcp, cleanup, 128, dropPrivs); + } + + version(cgi_use_fiber) + if(useFork) { + foreach(listener; listeners) + listener.blocking = false; + } + + // this is the UI control thread and thus gets more priority + Thread.getThis.priority = Thread.PRIORITY_MAX; + } + + Socket[] listeners; + void delegate(Socket) handler; + + immutable bool useFork; + int numberOfThreads; +} + +Socket startListening(string host, ushort port, ref bool tcp, ref void delegate() cleanup, int backQueue, void delegate() dropPrivs) { + Socket listener; + if(host.startsWith("unix:")) { + version(Posix) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + cloexec(listener); + string filename = host["unix:".length .. $].idup; + listener.bind(new UnixAddress(filename)); + cleanup = delegate() { + listener.close(); + import std.file; + remove(filename); + }; + tcp = false; + } else { + throw new Exception("unix sockets not supported on this system"); + } + } else if(host.startsWith("abstract:")) { + version(linux) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + cloexec(listener); + string filename = "\0" ~ host["abstract:".length .. $]; + import std.stdio; stderr.writeln("Listening to abstract unix domain socket: ", host["abstract:".length .. $]); + listener.bind(new UnixAddress(filename)); + tcp = false; + } else { + throw new Exception("abstract unix sockets not supported on this system"); + } + } else { + auto address = host.length ? parseAddress(host, port) : new InternetAddress(port); + version(cgi_use_fiber) { + version(Windows) + listener = new PseudoblockingOverlappedSocket(AddressFamily.INET, SocketType.STREAM); + else + listener = new Socket(address.addressFamily, SocketType.STREAM); + } else { + listener = new Socket(address.addressFamily, SocketType.STREAM); + } + cloexec(listener); + listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); + if(address.addressFamily == AddressFamily.INET6) + listener.setOption(SocketOptionLevel.IPV6, SocketOption.IPV6_V6ONLY, true); + listener.bind(address); + cleanup = delegate() { + listener.close(); + }; + tcp = true; + } + + listener.listen(backQueue); + + if (dropPrivs !is null) // can be null, backwards compatibility + dropPrivs(); + + return listener; +} + +// helper function to send a lot to a socket. Since this blocks for the buffer (possibly several times), you should probably call it in a separate thread or something. +void sendAll(Socket s, const(void)[] data, string file = __FILE__, size_t line = __LINE__) { + if(data.length == 0) return; + ptrdiff_t amount; + //import std.stdio; writeln("***",cast(string) data,"///"); + do { + amount = s.send(data); + if(amount == Socket.ERROR) { + version(cgi_use_fiber) { + if(wouldHaveBlocked()) { + bool registered = true; + registerEventWakeup(®istered, s, WakeupEvent.Write); + continue; + } + } + throw new ConnectionException(s, lastSocketError, file, line); + } + assert(amount > 0); + + data = data[amount .. $]; + } while(data.length); +} + +class ConnectionException : Exception { + Socket socket; + this(Socket s, string msg, string file = __FILE__, size_t line = __LINE__) { + this.socket = s; + super(msg, file, line); + } +} + +class HttpVersionNotSupportedException : Exception { + this(string file = __FILE__, size_t line = __LINE__) { + super("HTTP Version Not Supported", file, line); + } +} + +alias void delegate(Socket) CMT; + +import core.thread; +/+ + cgi.d now uses a hybrid of event i/o and threads at the top level. + + Top level thread is responsible for accepting sockets and selecting on them. + + It then indicates to a child that a request is pending, and any random worker + thread that is free handles it. It goes into blocking mode and handles that + http request to completion. + + At that point, it goes back into the waiting queue. + + + This concept is only implemented on Linux. On all other systems, it still + uses the worker threads and semaphores (which is perfectly fine for a lot of + things! Just having a great number of keep-alive connections will break that.) + + + So the algorithm is: + + select(accept, event, pending) + if accept -> send socket to free thread, if any. if not, add socket to queue + if event -> send the signaling thread a socket from the queue, if not, mark it free + - event might block until it can be *written* to. it is a fifo sending socket fds! + + A worker only does one http request at a time, then signals its availability back to the boss. + + The socket the worker was just doing should be added to the one-off epoll read. If it is closed, + great, we can get rid of it. Otherwise, it is considered `pending`. The *kernel* manages that; the + actual FD will not be kept out here. + + So: + queue = sockets we know are ready to read now, but no worker thread is available + idle list = worker threads not doing anything else. they signal back and forth + + the workers all read off the event fd. This is the semaphore wait + + the boss waits on accept or other sockets read events (one off! and level triggered). If anything happens wrt ready read, + it puts it in the queue and writes to the event fd. + + The child could put the socket back in the epoll thing itself. + + The child needs to be able to gracefully handle being given a socket that just closed with no work. ++/ +class ConnectionThread : Thread { + this(ListeningConnectionManager lcm, CMT dg, int myThreadNumber) { + this.lcm = lcm; + this.dg = dg; + this.myThreadNumber = myThreadNumber; + super(&run); + } + + void run() { + while(true) { + // so if there's a bunch of idle keep-alive connections, it can + // consume all the worker threads... just sitting there. + lcm.semaphore.wait(); + if(globalStopFlag) + return; + Socket socket; + synchronized(lcm) { + auto idx = lcm.nextIndexFront; + socket = lcm.queue[idx]; + lcm.queue[idx] = null; + atomicOp!"+="(lcm.nextIndexFront, 1); + atomicOp!"-="(lcm.queueLength, 1); + } + try { + //import std.stdio; writeln(myThreadNumber, " taking it"); + dg(socket); + /+ + if(socket.isAlive) { + // process it more later + version(linux) { + import core.sys.linux.epoll; + epoll_event ev; + ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET; + ev.data.fd = socket.handle; + import std.stdio; writeln("adding"); + if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_ADD, socket.handle, &ev) == -1) { + if(errno == EEXIST) { + ev.events = EPOLLIN | EPOLLONESHOT | EPOLLET; + ev.data.fd = socket.handle; + if(epoll_ctl(lcm.epoll_fd, EPOLL_CTL_MOD, socket.handle, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } else + throw new Exception("epoll_ctl " ~ to!string(errno)); + } + //import std.stdio; writeln("keep alive"); + // writing to this private member is to prevent the GC from closing my precious socket when I'm trying to use it later + __traits(getMember, socket, "sock") = cast(socket_t) -1; + } else { + continue; // hope it times out in a reasonable amount of time... + } + } + +/ + } catch(ConnectionClosedException e) { + // can just ignore this, it is fairly normal + socket.close(); + } catch(Throwable e) { + import std.stdio; stderr.rawWrite(e.toString); stderr.rawWrite("\n"); + socket.close(); + } + } + } + + ListeningConnectionManager lcm; + CMT dg; + int myThreadNumber; +} + +version(cgi_use_fiber) +class WorkerThread : Thread { + this(ListeningConnectionManager lcm, CMT dg, int myThreadNumber) { + this.lcm = lcm; + this.dg = dg; + this.myThreadNumber = myThreadNumber; + super(&run); + } + + version(Windows) + void run() { + auto timeout = INFINITE; + PseudoblockingOverlappedSocket key; + OVERLAPPED* overlapped; + DWORD bytes; + while(!globalStopFlag && GetQueuedCompletionStatus(iocp, &bytes, cast(PULONG_PTR) &key, &overlapped, timeout)) { + if(key is null) + continue; + key.lastAnswer = bytes; + if(key.fiber) { + key.fiber.proceed(); + } else { + // we have a new connection, issue the first receive on it and issue the next accept + + auto sn = key.accepted; + + key.accept(); + + cloexec(sn); + if(lcm.tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + + dg(sn); + } + } + //SleepEx(INFINITE, TRUE); + } + + version(linux) + void run() { + + import core.sys.linux.epoll; + epfd = epoll_create1(EPOLL_CLOEXEC); + if(epfd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(exit) { + import core.sys.posix.unistd; + close(epfd); + } + + { + epoll_event ev; + ev.events = EPOLLIN; + ev.data.fd = cancelfd; + epoll_ctl(epfd, EPOLL_CTL_ADD, cancelfd, &ev); + } + + foreach(listener; lcm.listeners) { + epoll_event ev; + ev.events = EPOLLIN | EPOLLEXCLUSIVE; // EPOLLEXCLUSIVE is only available on kernels since like 2017 but that's prolly good enough. + ev.data.fd = listener.handle; + if(epoll_ctl(epfd, EPOLL_CTL_ADD, listener.handle, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } + + + + while(!globalStopFlag) { + Socket sn; + + epoll_event[64] events; + auto nfds = epoll_wait(epfd, events.ptr, events.length, -1); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + + outer: foreach(idx; 0 .. nfds) { + auto flags = events[idx].events; + + if(cast(size_t) events[idx].data.ptr == cast(size_t) cancelfd) { + globalStopFlag = true; + //import std.stdio; writeln("exit heard"); + break; + } else { + foreach(listener; lcm.listeners) { + if(cast(size_t) events[idx].data.ptr == cast(size_t) listener.handle) { + //import std.stdio; writeln(myThreadNumber, " woken up ", flags); + // this try/catch is because it is set to non-blocking mode + // and Phobos' stupid api throws an exception instead of returning + // if it would block. Why would it block? because a forked process + // might have beat us to it, but the wakeup event thundered our herds. + try + sn = listener.accept(); // don't need to do the acceptCancelable here since the epoll checks it better + catch(SocketAcceptException e) { continue outer; } + + cloexec(sn); + if(lcm.tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + + sn.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10)); + } + + dg(sn); + continue outer; + } else { + // writeln(events[idx].data.ptr); + } + } + + if(cast(size_t) events[idx].data.ptr < 1024) { + throw arsd.core.ArsdException!"this doesn't look like a fiber pointer... "(cast(size_t) events[idx].data.ptr); + } + auto fiber = cast(CgiFiber) events[idx].data.ptr; + fiber.proceed(); + } + } + } + } + + ListeningConnectionManager lcm; + CMT dg; + int myThreadNumber; +} + + +/* Done with network helper */ + +/* Helpers for doing temporary files. Used both here and in web.d */ + +version(Windows) { + import core.sys.windows.windows; + extern(Windows) DWORD GetTempPathW(DWORD, LPWSTR); + alias GetTempPathW GetTempPath; +} + +version(Posix) { + static import linux = core.sys.posix.unistd; +} + +string getTempDirectory() { + string path; + version(Windows) { + wchar[1024] buffer; + auto len = GetTempPath(1024, buffer.ptr); + if(len == 0) + throw new Exception("couldn't find a temporary path"); + + auto b = buffer[0 .. len]; + + path = to!string(b); + } else + path = "/tmp/"; + + return path; +} + + +// I like std.date. These functions help keep my old code and data working with phobos changing. + +long sysTimeToDTime(in SysTime sysTime) { + return convert!("hnsecs", "msecs")(sysTime.stdTime - 621355968000000000L); +} + +long dateTimeToDTime(in DateTime dt) { + return sysTimeToDTime(cast(SysTime) dt); +} + +long getUtcTime() { // renamed primarily to avoid conflict with std.date itself + return sysTimeToDTime(Clock.currTime(UTC())); +} + +// NOTE: new SimpleTimeZone(minutes); can perhaps work with the getTimezoneOffset() JS trick +SysTime dTimeToSysTime(long dTime, immutable TimeZone tz = null) { + immutable hnsecs = convert!("msecs", "hnsecs")(dTime) + 621355968000000000L; + return SysTime(hnsecs, tz); +} + + + +// this is a helper to read HTTP transfer-encoding: chunked responses +immutable(ubyte[]) dechunk(BufferedInputRange ir) { + immutable(ubyte)[] ret; + + another_chunk: + // If here, we are at the beginning of a chunk. + auto a = ir.front(); + int chunkSize; + int loc = locationOf(a, "\r\n"); + while(loc == -1) { + ir.popFront(); + a = ir.front(); + loc = locationOf(a, "\r\n"); + } + + string hex; + hex = ""; + for(int i = 0; i < loc; i++) { + char c = a[i]; + if(c >= 'A' && c <= 'Z') + c += 0x20; + if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) { + hex ~= c; + } else { + break; + } + } + + assert(hex.length); + + int power = 1; + int size = 0; + foreach(cc1; retro(hex)) { + dchar cc = cc1; + if(cc >= 'a' && cc <= 'z') + cc -= 0x20; + int val = 0; + if(cc >= '0' && cc <= '9') + val = cc - '0'; + else + val = cc - 'A' + 10; + + size += power * val; + power *= 16; + } + + chunkSize = size; + assert(size >= 0); + + if(loc + 2 > a.length) { + ir.popFront(0, a.length + loc + 2); + a = ir.front(); + } + + a = ir.consume(loc + 2); + + if(chunkSize == 0) { // we're done with the response + // if we got here, will change must be true.... + more_footers: + loc = locationOf(a, "\r\n"); + if(loc == -1) { + ir.popFront(); + a = ir.front; + goto more_footers; + } else { + assert(loc == 0); + ir.consume(loc + 2); + goto finish; + } + } else { + // if we got here, will change must be true.... + if(a.length < chunkSize + 2) { + ir.popFront(0, chunkSize + 2); + a = ir.front(); + } + + ret ~= (a[0..chunkSize]); + + if(!(a.length > chunkSize + 2)) { + ir.popFront(0, chunkSize + 2); + a = ir.front(); + } + assert(a[chunkSize] == 13); + assert(a[chunkSize+1] == 10); + a = ir.consume(chunkSize + 2); + chunkSize = 0; + goto another_chunk; + } + + finish: + return ret; +} + +// I want to be able to get data from multiple sources the same way... +interface ByChunkRange { + bool empty(); + void popFront(); + const(ubyte)[] front(); +} + +ByChunkRange byChunk(const(ubyte)[] data) { + return new class ByChunkRange { + override bool empty() { + return !data.length; + } + + override void popFront() { + if(data.length > 4096) + data = data[4096 .. $]; + else + data = null; + } + + override const(ubyte)[] front() { + return data[0 .. $ > 4096 ? 4096 : $]; + } + }; +} + +ByChunkRange byChunk(BufferedInputRange ir, size_t atMost) { + const(ubyte)[] f; + + f = ir.front; + if(f.length > atMost) + f = f[0 .. atMost]; + + return new class ByChunkRange { + override bool empty() { + return atMost == 0; + } + + override const(ubyte)[] front() { + return f; + } + + override void popFront() { + ir.consume(f.length); + atMost -= f.length; + auto a = ir.front(); + + if(a.length <= atMost) { + f = a; + atMost -= a.length; + a = ir.consume(a.length); + if(atMost != 0) + ir.popFront(); + if(f.length == 0) { + f = ir.front(); + } + } else { + // we actually have *more* here than we need.... + f = a[0..atMost]; + atMost = 0; + ir.consume(atMost); + } + } + }; +} + +version(cgi_with_websocket) { + // http://tools.ietf.org/html/rfc6455 + + /++ + WEBSOCKET SUPPORT: + + Full example: + --- + import arsd.cgi; + + void websocketEcho(Cgi cgi) { + if(cgi.websocketRequested()) { + if(cgi.origin != "http://arsdnet.net") + throw new Exception("bad origin"); + auto websocket = cgi.acceptWebsocket(); + + websocket.send("hello"); + websocket.send(" world!"); + + auto msg = websocket.recv(); + while(msg.opcode != WebSocketOpcode.close) { + if(msg.opcode == WebSocketOpcode.text) { + websocket.send(msg.textData); + } else if(msg.opcode == WebSocketOpcode.binary) { + websocket.send(msg.data); + } + + msg = websocket.recv(); + } + + websocket.close(); + } else { + cgi.write("You are loading the websocket endpoint in a browser instead of a websocket client. Use a websocket client on this url instead.\n", true); + } + } + + mixin GenericMain!websocketEcho; + --- + +/ + + class WebSocket { + Cgi cgi; + + private bool isClient = false; + + private this(Cgi cgi) { + this.cgi = cgi; + + Socket socket = cgi.idlol.source; + socket.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"minutes"(5)); + } + + // returns true if data available, false if it timed out + bool recvAvailable(Duration timeout = dur!"msecs"(0)) { + if(!waitForNextMessageWouldBlock()) + return true; + if(isDataPending(timeout)) + return true; // this is kinda a lie. + + return false; + } + + public bool lowLevelReceive() { + auto bfr = cgi.idlol; + top: + auto got = bfr.front; + if(got.length) { + if(receiveBuffer.length < receiveBufferUsedLength + got.length) + receiveBuffer.length += receiveBufferUsedLength + got.length; + + receiveBuffer[receiveBufferUsedLength .. receiveBufferUsedLength + got.length] = got[]; + receiveBufferUsedLength += got.length; + bfr.consume(got.length); + + return true; + } + + if(bfr.sourceClosed) + return false; + + bfr.popFront(0); + if(bfr.sourceClosed) + return false; + goto top; + } + + + bool isDataPending(Duration timeout = 0.seconds) { + Socket socket = cgi.idlol.source; + + auto check = new SocketSet(); + check.add(socket); + + auto got = Socket.select(check, null, null, timeout); + if(got > 0) + return true; + return false; + } + + // note: this blocks + WebSocketFrame recv() { + return waitForNextMessage(); + } + + + + + private void llclose() { + cgi.close(); + } + + private void llsend(ubyte[] data) { + cgi.write(data); + cgi.flush(); + } + + void unregisterActiveSocket(WebSocket) {} + + /* copy/paste section { */ + + private int readyState_; + private ubyte[] receiveBuffer; + private size_t receiveBufferUsedLength; + + private Config config; + + enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. + enum OPEN = 1; /// The connection is open and ready to communicate. + enum CLOSING = 2; /// The connection is in the process of closing. + enum CLOSED = 3; /// The connection is closed or couldn't be opened. + + /++ + + +/ + /// Group: foundational + static struct Config { + /++ + These control the size of the receive buffer. + + It starts at the initial size, will temporarily + balloon up to the maximum size, and will reuse + a buffer up to the likely size. + + Anything larger than the maximum size will cause + the connection to be aborted and an exception thrown. + This is to protect you against a peer trying to + exhaust your memory, while keeping the user-level + processing simple. + +/ + size_t initialReceiveBufferSize = 4096; + size_t likelyReceiveBufferSize = 4096; /// ditto + size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + + /++ + Maximum combined size of a message. + +/ + size_t maximumMessageSize = 10 * 1024 * 1024; + + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; + string origin; /// Origin URL to send with the handshake, if desired. + string protocol; /// the protocol header, if desired. + + /++ + Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: + + --- + Config config; + config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; + --- + + History: + Added February 19, 2021 (included in dub version 9.2) + +/ + string[] additionalHeaders; + + /++ + Amount of time (in msecs) of idleness after which to send an automatic ping + + Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that + keeps the socket alive. + +/ + int pingFrequency = 5000; + + /++ + Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. + + The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! + + History: + Added March 31, 2021 (included in dub version 9.4) + +/ + Duration timeoutFromInactivity = 1.minutes; + + /++ + For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be + verified. Setting this to `false` will skip this check and allow the connection to continue anyway. + + History: + Added April 5, 2022 (dub v10.8) + + Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes + even if it was true, it would skip the verification. Now, it always respects this local setting. + +/ + bool verifyPeer = true; + } + + /++ + Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. + +/ + int readyState() { + return readyState_; + } + + /++ + Closes the connection, sending a graceful teardown message to the other side. + + Code 1000 is the normal closure code. + + History: + The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, + but also ignored anyway. + +/ + /// Group: foundational + void close(int code = 1000, string reason = null) + //in (reason.length < 123) + in { assert(reason.length < 123); } do + { + if(readyState_ != OPEN) + return; // it cool, we done + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.close; + wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; + wss.send(&llsend); + + readyState_ = CLOSING; + + closeCalled = true; + + llclose(); + } + + private bool closeCalled; + + /++ + Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. + +/ + /// Group: foundational + void ping(in ubyte[] data = null) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.ping; + if(data !is null) wss.data = data.dup; + wss.send(&llsend); + } + + /++ + Sends a pong message to the server. This is normally done automatically in response to pings. + +/ + /// Group: foundational + void pong(in ubyte[] data = null) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.pong; + if(data !is null) wss.data = data.dup; + wss.send(&llsend); + } + + /++ + Sends a text message through the websocket. + +/ + /// Group: foundational + void send(in char[] textData) { + WebSocketFrame wss; + wss.fin = true; + wss.masked = this.isClient; + wss.opcode = WebSocketOpcode.text; + wss.data = cast(ubyte[]) textData.dup; + wss.send(&llsend); + } + + /++ + Sends a binary message through the websocket. + +/ + /// Group: foundational + void send(in ubyte[] binaryData) { + WebSocketFrame wss; + wss.masked = this.isClient; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData.dup; + wss.send(&llsend); + } + + /++ + Waits for and returns the next complete message on the socket. + + Note that the onmessage function is still called, right before + this returns. + +/ + /// Group: blocking_api + public WebSocketFrame waitForNextMessage() { + do { + auto m = processOnce(); + if(m.populated) + return m; + } while(lowLevelReceive()); + + throw new ConnectionClosedException("Websocket receive timed out"); + //return WebSocketFrame.init; // FIXME? maybe. + } + + /++ + Tells if [waitForNextMessage] would block. + +/ + /// Group: blocking_api + public bool waitForNextMessageWouldBlock() { + checkAgain: + if(isMessageBuffered()) + return false; + if(!isDataPending()) + return true; + + while(isDataPending()) { + if(lowLevelReceive() == false) + throw new ConnectionClosedException("Connection closed in middle of message"); + } + + goto checkAgain; + } + + /++ + Is there a message in the buffer already? + If `true`, [waitForNextMessage] is guaranteed to return immediately. + If `false`, check [isDataPending] as the next step. + +/ + /// Group: blocking_api + public bool isMessageBuffered() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + if(d.length) { + auto orig = d; + auto m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d !is orig) + return true; + } + + return false; + } + + private ubyte continuingType; + private ubyte[] continuingData; + //private size_t continuingDataLength; + + private WebSocketFrame processOnce() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. + WebSocketFrame m; + if(d.length) { + auto orig = d; + m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d is orig) + return WebSocketFrame.init; + m.unmaskInPlace(); + switch(m.opcode) { + case WebSocketOpcode.continuation: + if(continuingData.length + m.data.length > config.maximumMessageSize) + throw new Exception("message size exceeded"); + + continuingData ~= m.data; + if(m.fin) { + if(ontextmessage) + ontextmessage(cast(char[]) continuingData); + if(onbinarymessage) + onbinarymessage(continuingData); + + continuingData = null; + } + break; + case WebSocketOpcode.text: + if(m.fin) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.binary: + if(m.fin) { + if(onbinarymessage) + onbinarymessage(m.data); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.close: + + //import std.stdio; writeln("closed ", cast(string) m.data); + + ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; + const(char)[] reason; + + if(m.data.length >= 2) { + code = (m.data[0] << 8) | m.data[1]; + reason = (cast(char[]) m.data[2 .. $]); + } + + if(onclose) + onclose(CloseEvent(code, reason, true)); + + // if we receive one and haven't sent one back we're supposed to echo it back and close. + if(!closeCalled) + close(code, reason.idup); + + readyState_ = CLOSED; + + unregisterActiveSocket(this); + break; + case WebSocketOpcode.ping: + // import std.stdio; writeln("ping received ", m.data); + pong(m.data); + break; + case WebSocketOpcode.pong: + // import std.stdio; writeln("pong received ", m.data); + // just really references it is still alive, nbd. + break; + default: // ignore though i could and perhaps should throw too + } + } + + if(d.length) { + m.data = m.data.dup(); + } + + import core.stdc.string; + memmove(receiveBuffer.ptr, d.ptr, d.length); + receiveBufferUsedLength = d.length; + + return m; + } + + private void autoprocess() { + // FIXME + do { + processOnce(); + } while(lowLevelReceive()); + } + + /++ + Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. + + $(PITFALL + The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. + ) + + History: + Added March 19, 2023 (dub v11.0). + +/ + static struct CloseEvent { + ushort code; + const(char)[] reason; + bool wasClean; + + string extendedErrorInformationUnstable; + + /++ + See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. + +/ + enum StandardCloseCodes { + purposeFulfilled = 1000, + goingAway = 1001, + protocolError = 1002, + unacceptableData = 1003, // e.g. got text message when you can only handle binary + Reserved = 1004, + noStatusCodePresent = 1005, // not set by endpoint. + abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these + inconsistentData = 1007, // e.g. utf8 validation failed + genericPolicyViolation = 1008, + messageTooBig = 1009, + clientRequiredExtensionMissing = 1010, // only the client should send this + unnexpectedCondition = 1011, + unverifiedCertificate = 1015, // not set by client + } + } + + /++ + The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. + + History: + The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. + + Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. + +/ + arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose; + void delegate() onerror; /// + void delegate(in char[]) ontextmessage; /// + void delegate(in ubyte[]) onbinarymessage; /// + void delegate() onopen; /// + + /++ + + +/ + /// Group: browser_api + void onmessage(void delegate(in char[]) dg) { + ontextmessage = dg; + } + + /// ditto + void onmessage(void delegate(in ubyte[]) dg) { + onbinarymessage = dg; + } + + /* } end copy/paste */ + + + + } + + /++ + Returns true if the request headers are asking for a websocket upgrade. + + If this returns true, and you want to accept it, call [acceptWebsocket]. + +/ + bool websocketRequested(Cgi cgi) { + return + "sec-websocket-key" in cgi.requestHeaders + && + "connection" in cgi.requestHeaders && + cgi.requestHeaders["connection"].asLowerCase().canFind("upgrade") + && + "upgrade" in cgi.requestHeaders && + cgi.requestHeaders["upgrade"].asLowerCase().equal("websocket") + ; + } + + /++ + If [websocketRequested], you can call this to accept it and upgrade the connection. It returns the new [WebSocket] object you use for future communication on this connection; the `cgi` object should no longer be used. + +/ + WebSocket acceptWebsocket(Cgi cgi) { + assert(!cgi.closed); + assert(!cgi.outputtedResponseData); + cgi.setResponseStatus("101 Switching Protocols"); + cgi.header("Upgrade: WebSocket"); + cgi.header("Connection: upgrade"); + + string key = cgi.requestHeaders["sec-websocket-key"]; + key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; // the defined guid from the websocket spec + + import std.digest.sha; + auto hash = sha1Of(key); + auto accept = Base64.encode(hash); + + cgi.header(("Sec-WebSocket-Accept: " ~ accept).idup); + + cgi.websocketMode = true; + cgi.write(""); + + cgi.flush(); + + auto ws = new WebSocket(cgi); + ws.readyState_ = WebSocket.OPEN; + return ws; + } + + // FIXME get websocket to work on other modes, not just embedded_httpd + + /* copy/paste in http2.d { */ + enum WebSocketOpcode : ubyte { + continuation = 0, + text = 1, + binary = 2, + // 3, 4, 5, 6, 7 RESERVED + close = 8, + ping = 9, + pong = 10, + // 11,12,13,14,15 RESERVED + } + + public struct WebSocketFrame { + private bool populated; + bool fin; + bool rsv1; + bool rsv2; + bool rsv3; + WebSocketOpcode opcode; // 4 bits + bool masked; + ubyte lengthIndicator; // don't set this when building one to send + ulong realLength; // don't use when sending + ubyte[4] maskingKey; // don't set this when sending + ubyte[] data; + + static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { + WebSocketFrame msg; + msg.fin = true; + msg.opcode = opcode; + msg.data = cast(ubyte[]) data.dup; + + return msg; + } + + private void send(scope void delegate(ubyte[]) llsend) { + ubyte[64] headerScratch; + int headerScratchPos = 0; + + realLength = data.length; + + { + ubyte b1; + b1 |= cast(ubyte) opcode; + b1 |= rsv3 ? (1 << 4) : 0; + b1 |= rsv2 ? (1 << 5) : 0; + b1 |= rsv1 ? (1 << 6) : 0; + b1 |= fin ? (1 << 7) : 0; + + headerScratch[0] = b1; + headerScratchPos++; + } + + { + headerScratchPos++; // we'll set header[1] at the end of this + auto rlc = realLength; + ubyte b2; + b2 |= masked ? (1 << 7) : 0; + + assert(headerScratchPos == 2); + + if(realLength > 65535) { + // use 64 bit length + b2 |= 0x7f; + + // FIXME: double check endinaness + foreach(i; 0 .. 8) { + headerScratch[2 + 7 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 8; + } else if(realLength > 125) { + // use 16 bit length + b2 |= 0x7e; + + // FIXME: double check endinaness + foreach(i; 0 .. 2) { + headerScratch[2 + 1 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 2; + } else { + // use 7 bit length + b2 |= realLength & 0b_0111_1111; + } + + headerScratch[1] = b2; + } + + //assert(!masked, "masking key not properly implemented"); + if(masked) { + // FIXME: randomize this + headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; + headerScratchPos += 4; + + // we'll just mask it in place... + int keyIdx = 0; + foreach(i; 0 .. data.length) { + data[i] = data[i] ^ maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); + llsend(headerScratch[0 .. headerScratchPos]); + llsend(data); + } + + static WebSocketFrame read(ref ubyte[] d) { + WebSocketFrame msg; + + auto orig = d; + + WebSocketFrame needsMoreData() { + d = orig; + return WebSocketFrame.init; + } + + if(d.length < 2) + return needsMoreData(); + + ubyte b = d[0]; + + msg.populated = true; + + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); + b >>= 4; + msg.rsv3 = b & 0x01; + b >>= 1; + msg.rsv2 = b & 0x01; + b >>= 1; + msg.rsv1 = b & 0x01; + b >>= 1; + msg.fin = b & 0x01; + + b = d[1]; + msg.masked = (b & 0b1000_0000) ? true : false; + msg.lengthIndicator = b & 0b0111_1111; + + d = d[2 .. $]; + + if(msg.lengthIndicator == 0x7e) { + // 16 bit length + msg.realLength = 0; + + if(d.length < 2) return needsMoreData(); + + foreach(i; 0 .. 2) { + msg.realLength |= d[0] << ((1-i) * 8); + d = d[1 .. $]; + } + } else if(msg.lengthIndicator == 0x7f) { + // 64 bit length + msg.realLength = 0; + + if(d.length < 8) return needsMoreData(); + + foreach(i; 0 .. 8) { + msg.realLength |= ulong(d[0]) << ((7-i) * 8); + d = d[1 .. $]; + } + } else { + // 7 bit length + msg.realLength = msg.lengthIndicator; + } + + if(msg.masked) { + + if(d.length < 4) return needsMoreData(); + + msg.maskingKey = d[0 .. 4]; + d = d[4 .. $]; + } + + if(msg.realLength > d.length) { + return needsMoreData(); + } + + msg.data = d[0 .. cast(size_t) msg.realLength]; + d = d[cast(size_t) msg.realLength .. $]; + + return msg; + } + + void unmaskInPlace() { + if(this.masked) { + int keyIdx = 0; + foreach(i; 0 .. this.data.length) { + this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + } + + char[] textData() { + return cast(char[]) data; + } + } + /* } */ +} + + +version(Windows) +{ + version(CRuntime_DigitalMars) + { + extern(C) int setmode(int, int) nothrow @nogc; + } + else version(CRuntime_Microsoft) + { + extern(C) int _setmode(int, int) nothrow @nogc; + alias setmode = _setmode; + } + else static assert(0); +} + +version(Posix) { + import core.sys.posix.unistd; + version(CRuntime_Musl) {} else { + private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**); + } +} + + +// FIXME: these aren't quite public yet. +//private: + +// template for laziness +void startAddonServer()(string arg) { + version(OSX) { + assert(0, "Not implemented"); + } else version(linux) { + import core.sys.posix.unistd; + pid_t pid; + const(char)*[16] args; + args[0] = "ARSD_CGI_ADDON_SERVER"; + args[1] = arg.ptr; + posix_spawn(&pid, "/proc/self/exe", + null, + null, + args.ptr, + null // env + ); + } else version(Windows) { + wchar[2048] filename; + auto len = GetModuleFileNameW(null, filename.ptr, cast(DWORD) filename.length); + if(len == 0 || len == filename.length) + throw new Exception("could not get process name to start helper server"); + + STARTUPINFOW startupInfo; + startupInfo.cb = cast(DWORD) startupInfo.sizeof; + PROCESS_INFORMATION processInfo; + + import std.utf; + + // I *MIGHT* need to run it as a new job or a service... + auto ret = CreateProcessW( + filename.ptr, + toUTF16z(arg), + null, // process attributes + null, // thread attributes + false, // inherit handles + 0, // creation flags + null, // environment + null, // working directory + &startupInfo, + &processInfo + ); + + if(!ret) + throw new Exception("create process failed"); + + // when done with those, if we set them + /* + CloseHandle(hStdInput); + CloseHandle(hStdOutput); + CloseHandle(hStdError); + */ + + } else static assert(0, "Websocket server not implemented on this system yet (email me, i can prolly do it if you need it)"); +} + +// template for laziness +/* + The websocket server is a single-process, single-thread, event + I/O thing. It is passed websockets from other CGI processes + and is then responsible for handling their messages and responses. + Note that the CGI process is responsible for websocket setup, + including authentication, etc. + + It also gets data sent to it by other processes and is responsible + for distributing that, as necessary. +*/ +void runWebsocketServer()() { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(WebSocket ws, string group) { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(string content, string group) { + assert(0, "not implemented"); +} + + +void runEventServer()() { + runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServerImplementation()); +} + +void runTimerServer()() { + runAddonServer("/tmp/arsd_scheduled_job_server", new ScheduledJobServerImplementation()); +} + +version(Posix) { + alias LocalServerConnectionHandle = int; + alias CgiConnectionHandle = int; + alias SocketConnectionHandle = int; + + enum INVALID_CGI_CONNECTION_HANDLE = -1; +} else version(Windows) { + alias LocalServerConnectionHandle = HANDLE; + version(embedded_httpd_threads) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else version(fastcgi) { + alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point. + enum INVALID_CGI_CONNECTION_HANDLE = null; + } else version(scgi) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else { /* version(plain_cgi) */ + alias CgiConnectionHandle = HANDLE; + enum INVALID_CGI_CONNECTION_HANDLE = null; + } + alias SocketConnectionHandle = SOCKET; +} + +version(with_addon_servers_connections) +LocalServerConnectionHandle openLocalServerConnection()(string name, string arg) { + version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.sys.un; + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + cloexec(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + } + + bool alreadyTried; + + try_again: + + if(connect(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + if(!alreadyTried && errno == ECONNREFUSED) { + // try auto-spawning the server, then attempt connection again + startAddonServer(arg); + import core.thread; + Thread.sleep(50.msecs); + alreadyTried = true; + goto try_again; + } else + throw new Exception("connect " ~ to!string(errno)); + } + + return sock; + } else version(Windows) { + return null; // FIXME + } +} + +version(with_addon_servers_connections) +void closeLocalServerConnection(LocalServerConnectionHandle handle) { + version(Posix) { + import core.sys.posix.unistd; + close(handle); + } else version(Windows) + CloseHandle(handle); +} + +void runSessionServer()() { + runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation()); +} + +import core.stdc.errno; + +struct IoOp { + @disable this(); + @disable this(this); + + /* + So we want to be able to eventually handle generic sockets too. + */ + + enum Read = 1; + enum Write = 2; + enum Accept = 3; + enum ReadSocketHandle = 4; + + // Your handler may be called in a different thread than the one that initiated the IO request! + // It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution. + private bool delegate(IoOp*, int) handler; // returns true if you are done and want it to be closed + private void delegate(IoOp*) closeHandler; + private void delegate(IoOp*) completeHandler; + private int internalFd; + private int operation; + private int bufferLengthAllocated; + private int bufferLengthUsed; + private ubyte[1] internalBuffer; // it can be overallocated! + + ubyte[] allocatedBuffer() return { + return internalBuffer.ptr[0 .. bufferLengthAllocated]; + } + + ubyte[] usedBuffer() return { + return allocatedBuffer[0 .. bufferLengthUsed]; + } + + void reset() { + bufferLengthUsed = 0; + } + + int fd() { + return internalFd; + } +} + +IoOp* allocateIoOp(int fd, int operation, int bufferSize, bool delegate(IoOp*, int) handler) { + import core.stdc.stdlib; + + auto ptr = calloc(IoOp.sizeof + bufferSize, 1); + if(ptr is null) + assert(0); // out of memory! + + auto op = cast(IoOp*) ptr; + + op.handler = handler; + op.internalFd = fd; + op.operation = operation; + op.bufferLengthAllocated = bufferSize; + op.bufferLengthUsed = 0; + + import core.memory; + + GC.addRoot(ptr); + + return op; +} + +void freeIoOp(ref IoOp* ptr) { + + import core.memory; + GC.removeRoot(ptr); + + import core.stdc.stdlib; + free(ptr); + ptr = null; +} + +version(Posix) +version(with_addon_servers_connections) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + + //import std.stdio : writeln; writeln(cast(string) data); + + import core.sys.posix.unistd; + + auto ret = write(connection, data.ptr, data.length); + if(ret != data.length) { + if(ret == 0 || (ret == -1 && (errno == EPIPE || errno == ETIMEDOUT))) { + // the file is closed, remove it + eis.fileClosed(connection); + } else + throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME + } +} +version(Windows) +version(with_addon_servers_connections) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + // FIXME +} + +bool isInvalidHandle(CgiConnectionHandle h) { + return h == INVALID_CGI_CONNECTION_HANDLE; +} + +/+ +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv +https://support.microsoft.com/en-gb/help/181611/socket-overlapped-i-o-versus-blocking-nonblocking-mode +https://stackoverflow.com/questions/18018489/should-i-use-iocps-or-overlapped-wsasend-receive +https://docs.microsoft.com/en-us/windows/desktop/fileio/i-o-completion-ports +https://docs.microsoft.com/en-us/windows/desktop/fileio/createiocompletionport +https://docs.microsoft.com/en-us/windows/desktop/api/mswsock/nf-mswsock-acceptex +https://docs.microsoft.com/en-us/windows/desktop/Sync/waitable-timer-objects +https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-setwaitabletimer +https://docs.microsoft.com/en-us/windows/desktop/Sync/using-a-waitable-timer-with-an-asynchronous-procedure-call +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsagetoverlappedresult + ++/ + +/++ + You can customize your server by subclassing the appropriate server. Then, register your + subclass at compile time with the [registerEventIoServer] template, or implement your own + main function and call it yourself. + + $(TIP If you make your subclass a `final class`, there is a slight performance improvement.) ++/ +version(with_addon_servers_connections) +interface EventIoServer { + bool handleLocalConnectionData(IoOp* op, int receivedFd); + void handleLocalConnectionClose(IoOp* op); + void handleLocalConnectionComplete(IoOp* op); + void wait_timeout(); + void fileClosed(int fd); + + void epoll_fd(int fd); +} + +// the sink should buffer it +private void serialize(T)(scope void delegate(scope ubyte[]) sink, T t) { + static if(is(T == struct)) { + foreach(member; __traits(allMembers, T)) + serialize(sink, __traits(getMember, t, member)); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + sink((cast(ubyte*) &t)[0 .. t.sizeof]); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc. + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + sink(cast(ubyte[]) t[]); + } else static if(is(T : A[], A)) { + // generic array is less optimal but still prolly ok + int len = cast(int) t.length; + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + foreach(item; t) + serialize(sink, item); + } else static assert(0, T.stringof); +} + +// all may be stack buffers, so use cautio +private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) { + static if(is(T == struct)) { + T t; + foreach(member; __traits(allMembers, T)) + deserialize!(typeof(__traits(getMember, T, member)))(get, (mbr) { __traits(getMember, t, member) = mbr; }); + dg(t); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + T t; + auto data = get(t.sizeof); + t = (cast(T[]) data)[0]; + dg(t); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + + /* + typeof(T[0])[2000] stackBuffer; + T buffer; + + if(len < stackBuffer.length) + buffer = stackBuffer[0 .. len]; + else + buffer = new T(len); + + data = get(len * typeof(T[0]).sizeof); + */ + + T t = cast(T) get(len * cast(int) typeof(T.init[0]).sizeof); + + dg(t); + } else static if(is(T == E[], E)) { + T t; + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + t.length = len; + foreach(ref e; t) { + deserialize!E(get, (ele) { e = ele; }); + } + dg(t); + } else static assert(0, T.stringof); +} + +unittest { + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 1); }); + }, 1); + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 56674); }); + }, 56674); + ubyte[1000] buffer; + int bufferPoint; + void add(scope ubyte[] b) { + buffer[bufferPoint .. bufferPoint + b.length] = b[]; + bufferPoint += b.length; + } + ubyte[] get(int sz) { + auto b = buffer[bufferPoint .. bufferPoint + sz]; + bufferPoint += sz; + return b; + } + serialize(&add, "test here"); + bufferPoint = 0; + deserialize!string(&get, (t) { assert(t == "test here"); }); + bufferPoint = 0; + + struct Foo { + int a; + ubyte c; + string d; + } + serialize(&add, Foo(403, 37, "amazing")); + bufferPoint = 0; + deserialize!Foo(&get, (t) { + assert(t.a == 403); + assert(t.c == 37); + assert(t.d == "amazing"); + }); + bufferPoint = 0; +} + +/* + Here's the way the RPC interface works: + + You define the interface that lists the functions you can call on the remote process. + The interface may also have static methods for convenience. These forward to a singleton + instance of an auto-generated class, which actually sends the args over the pipe. + + An impl class actually implements it. A receiving server deserializes down the pipe and + calls methods on the class. + + I went with the interface to get some nice compiler checking and documentation stuff. + + I could have skipped the interface and just implemented it all from the server class definition + itself, but then the usage may call the method instead of rpcing it; I just like having the user + interface and the implementation separate so you aren't tempted to `new impl` to call the methods. + + + I fiddled with newlines in the mixin string to ensure the assert line numbers matched up to the source code line number. Idk why dmd didn't do this automatically, but it was important to me. + + Realistically though the bodies would just be + connection.call(this.mangleof, args...) sooooo. + + FIXME: overloads aren't supported +*/ + +/// Base for storing sessions in an array. Exists primarily for internal purposes and you should generally not use this. +interface SessionObject {} + +private immutable void delegate(string[])[string] scheduledJobHandlers; +private immutable void delegate(string[])[string] websocketServers; + +version(with_breaking_cgi_features) +mixin(q{ + +mixin template ImplementRpcClientInterface(T, string serverPath, string cmdArg) { + static import std.traits; + + // derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering. + static foreach(idx, member; __traits(derivedMembers, T)) { + static if(__traits(isVirtualMethod, __traits(getMember, T, member))) + mixin( q{ + std.traits.ReturnType!(__traits(getMember, T, member)) + } ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params) + { + SerializationBuffer buffer; + auto i = cast(ushort) idx; + serialize(&buffer.sink, i); + serialize(&buffer.sink, __traits(getMember, T, member).mangleof); + foreach(param; params) + serialize(&buffer.sink, param); + + auto sendable = buffer.sendable; + + version(Posix) {{ + auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0); + + if(ret == -1) { + throw new Exception("send returned -1, errno: " ~ to!string(errno)); + } else if(ret == 0) { + throw new Exception("Connection to addon server lost"); + } if(ret < sendable.length) + throw new Exception("Send failed to send all"); + assert(ret == sendable.length); + }} // FIXME Windows impl + + static if(!is(typeof(return) == void)) { + // there is a return value; we need to wait for it too + version(Posix) { + ubyte[3000] revBuffer; + auto ret = recv(connectionHandle, revBuffer.ptr, revBuffer.length, 0); + auto got = revBuffer[0 .. ret]; + + int dataLocation; + ubyte[] grab(int sz) { + auto dataLocation1 = dataLocation; + dataLocation += sz; + return got[dataLocation1 .. dataLocation]; + } + + typeof(return) retu; + deserialize!(typeof(return))(&grab, (a) { retu = a; }); + return retu; + } else { + // FIXME Windows impl + return typeof(return).init; + } + + } + }}); + } + + private static typeof(this) singletonInstance; + private LocalServerConnectionHandle connectionHandle; + + static typeof(this) connection() { + if(singletonInstance is null) { + singletonInstance = new typeof(this)(); + singletonInstance.connect(); + } + return singletonInstance; + } + + void connect() { + connectionHandle = openLocalServerConnection(serverPath, cmdArg); + } + + void disconnect() { + closeLocalServerConnection(connectionHandle); + } +} + +void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(is(Class : Interface)) { + ushort calledIdx; + string calledFunction; + + int dataLocation; + ubyte[] grab(int sz) { + if(sz == 0) assert(0); + auto d = data[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + again: + + deserialize!ushort(&grab, (a) { calledIdx = a; }); + deserialize!string(&grab, (a) { calledFunction = a; }); + + import std.traits; + + sw: switch(calledIdx) { + foreach(idx, memberName; __traits(derivedMembers, Interface)) + static if(__traits(isVirtualMethod, __traits(getMember, Interface, memberName))) { + case idx: + assert(calledFunction == __traits(getMember, Interface, memberName).mangleof); + + Parameters!(__traits(getMember, Interface, memberName)) params; + foreach(ref param; params) + deserialize!(typeof(param))(&grab, (a) { param = a; }); + + static if(is(ReturnType!(__traits(getMember, Interface, memberName)) == void)) { + __traits(getMember, this_, memberName)(params); + } else { + auto ret = __traits(getMember, this_, memberName)(params); + SerializationBuffer buffer; + serialize(&buffer.sink, ret); + + auto sendable = buffer.sendable; + + version(Posix) { + auto r = send(fd, sendable.ptr, sendable.length, 0); + if(r == -1) { + throw new Exception("send returned -1, errno: " ~ to!string(errno)); + } else if(r == 0) { + throw new Exception("Connection to addon client lost"); + } if(r < sendable.length) + throw new Exception("Send failed to send all"); + + } // FIXME Windows impl + } + break sw; + } + default: assert(0); + } + + if(dataLocation != data.length) + goto again; +} + + +private struct SerializationBuffer { + ubyte[2048] bufferBacking; + int bufferLocation; + void sink(scope ubyte[] data) { + bufferBacking[bufferLocation .. bufferLocation + data.length] = data[]; + bufferLocation += data.length; + } + + ubyte[] sendable() return { + return bufferBacking[0 .. bufferLocation]; + } +} + +/* + FIXME: + add a version command line arg + version data in the library + management gui as external program + + at server with event_fd for each run + use .mangleof in the at function name + + i think the at server will have to: + pipe args to the child + collect child output for logging + get child return value for logging + + on windows timers work differently. idk how to best combine with the io stuff. + + will have to have dump and restore too, so i can restart without losing stuff. +*/ + +/++ + A convenience object for talking to the [BasicDataServer] from a higher level. + See: [Cgi.getSessionObject]. + + You pass it a `Data` struct describing the data you want saved in the session. + Then, this class will generate getter and setter properties that allow access + to that data. + + Note that each load and store will be done as-accessed; it doesn't front-load + mutable data nor does it batch updates out of fear of read-modify-write race + conditions. (In fact, right now it does this for everything, but in the future, + I might batch load `immutable` members of the Data struct.) + + At some point in the future, I might also let it do different backends, like + a client-side cookie store too, but idk. + + Note that the plain-old-data members of your `Data` struct are wrapped by this + interface via a static foreach to make property functions. + + See_Also: [MockSession] ++/ +interface Session(Data) : SessionObject { + @property string sessionId() const; + + /++ + Starts a new session. Note that a session is also + implicitly started as soon as you write data to it, + so if you need to alter these parameters from their + defaults, be sure to explicitly call this BEFORE doing + any writes to session data. + + Params: + idleLifetime = How long, in seconds, the session + should remain in memory when not being read from + or written to. The default is one day. + + NOT IMPLEMENTED + + useExtendedLifetimeCookie = The session ID is always + stored in a HTTP cookie, and by default, that cookie + is discarded when the user closes their browser. + + But if you set this to true, it will use a non-perishable + cookie for the given idleLifetime. + + NOT IMPLEMENTED + +/ + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false); + + /++ + Regenerates the session ID and updates the associated + cookie. + + This is also your chance to change immutable data + (not yet implemented). + +/ + void regenerateId(); + + /++ + Terminates this session, deleting all saved data. + +/ + void terminate(); + + /++ + Plain-old-data members of your `Data` struct are wrapped here via + the property getters and setters. + + If the member is a non-string array, it returns a magical array proxy + object which allows for atomic appends and replaces via overloaded operators. + You can slice this to get a range representing a $(B const) view of the array. + This is to protect you against read-modify-write race conditions. + +/ + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout; + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value); + }); + +} + +/++ + An implementation of [Session] that works on real cgi connections utilizing the + [BasicDataServer]. + + As opposed to a [MockSession] which is made for testing purposes. + + You will not construct one of these directly. See [Cgi.getSessionObject] instead. ++/ +class BasicDataServerSession(Data) : Session!Data { + private Cgi cgi; + private string sessionId_; + + public @property string sessionId() const { + return sessionId_; + } + + protected @property string sessionId(string s) { + return this.sessionId_ = s; + } + + private this(Cgi cgi) { + this.cgi = cgi; + if(auto ptr = "sessionId" in cgi.cookies) + sessionId = (*ptr).length ? *ptr : null; + } + + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) { + assert(sessionId is null); + + // FIXME: what if there is a session ID cookie, but no corresponding session on the server? + + import std.random, std.conv; + sessionId = to!string(uniform(1, long.max)); + + BasicDataServer.connection.createSession(sessionId, idleLifetime); + setCookie(); + } + + protected void setCookie() { + cgi.setCookie( + "sessionId", sessionId, + 0 /* expiration */, + "/" /* path */, + null /* domain */, + true /* http only */, + cgi.https /* if the session is started on https, keep it there, otherwise, be flexible */); + } + + void regenerateId() { + if(sessionId is null) { + start(); + return; + } + import std.random, std.conv; + auto oldSessionId = sessionId; + sessionId = to!string(uniform(1, long.max)); + BasicDataServer.connection.renameSession(oldSessionId, sessionId); + setCookie(); + } + + void terminate() { + BasicDataServer.connection.destroySession(sessionId); + sessionId = null; + setCookie(); + } + + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout { + if(sessionId is null) + return typeof(return).init; + + import std.traits; + auto v = BasicDataServer.connection.getSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName); + if(v.length == 0) + return typeof(return).init; + import std.conv; + // why this cast? to doesn't like being given an inout argument. so need to do it without that, then + // we need to return it and that needed the cast. It should be fine since we basically respect constness.. + // basically. Assuming the session is POD this should be fine. + return cast(typeof(return)) to!(typeof(__traits(getMember, Data, memberName)))(v); + } + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) { + if(sessionId is null) + start(); + import std.conv; + import std.traits; + BasicDataServer.connection.setSessionData(sessionId, fullyQualifiedName!Data ~ "." ~ memberName, to!string(value)); + return value; + } + }); +} + +/++ + A mock object that works like the real session, but doesn't actually interact with any actual database or http connection. + Simply stores the data in its instance members. ++/ +class MockSession(Data) : Session!Data { + pure { + @property string sessionId() const { return "mock"; } + void start(int idleLifetime = 2600 * 24, bool useExtendedLifetimeCookie = false) {} + void regenerateId() {} + void terminate() {} + + private Data store_; + + static foreach(memberName; __traits(allMembers, Data)) + static if(is(typeof(__traits(getMember, Data, memberName)))) + mixin(q{ + @property inout(typeof(__traits(getMember, Data, memberName))) } ~ memberName ~ q{ () inout { + return __traits(getMember, store_, memberName); + } + @property typeof(__traits(getMember, Data, memberName)) } ~ memberName ~ q{ (typeof(__traits(getMember, Data, memberName)) value) { + return __traits(getMember, store_, memberName) = value; + } + }); + } +} + +/++ + Direct interface to the basic data add-on server. You can + typically use [Cgi.getSessionObject] as a more convenient interface. ++/ +version(with_addon_servers_connections) +interface BasicDataServer { + /// + void createSession(string sessionId, int lifetime); + /// + void renewSession(string sessionId, int lifetime); + /// + void destroySession(string sessionId); + /// + void renameSession(string oldSessionId, string newSessionId); + + /// + void setSessionData(string sessionId, string dataKey, string dataValue); + /// + string getSessionData(string sessionId, string dataKey); + + /// + static BasicDataServerConnection connection() { + return BasicDataServerConnection.connection(); + } +} + +version(with_addon_servers_connections) +class BasicDataServerConnection : BasicDataServer { + mixin ImplementRpcClientInterface!(BasicDataServer, "/tmp/arsd_session_server", "--session-server"); +} + +version(with_addon_servers) +final class BasicDataServerImplementation : BasicDataServer, EventIoServer { + + void createSession(string sessionId, int lifetime) { + sessions[sessionId.idup] = Session(lifetime); + } + void destroySession(string sessionId) { + sessions.remove(sessionId); + } + void renewSession(string sessionId, int lifetime) { + sessions[sessionId].lifetime = lifetime; + } + void renameSession(string oldSessionId, string newSessionId) { + sessions[newSessionId.idup] = sessions[oldSessionId]; + sessions.remove(oldSessionId); + } + void setSessionData(string sessionId, string dataKey, string dataValue) { + if(sessionId !in sessions) + createSession(sessionId, 3600); // FIXME? + sessions[sessionId].values[dataKey.idup] = dataValue.idup; + } + string getSessionData(string sessionId, string dataKey) { + if(auto session = sessionId in sessions) { + if(auto data = dataKey in (*session).values) + return *data; + else + return null; // no such data + + } else { + return null; // no session + } + } + + + protected: + + struct Session { + int lifetime; + + string[string] values; + } + + Session[string] sessions; + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!BasicDataServer(this, data, op.fd); + return false; + } + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) {} // stateless so irrelevant + void epoll_fd(int fd) {} +} + +/++ + See [schedule] to make one of these. You then call one of the methods here to set it up: + + --- + schedule!fn(args).at(DateTime(2019, 8, 7, 12, 00, 00)); // run the function at August 7, 2019, 12 noon UTC + schedule!fn(args).delay(6.seconds); // run it after waiting 6 seconds + schedule!fn(args).asap(); // run it in the background as soon as the event loop gets around to it + --- ++/ +version(with_addon_servers_connections) +struct ScheduledJobHelper { + private string func; + private string[] args; + private bool consumed; + + private this(string func, string[] args) { + this.func = func; + this.args = args; + } + + ~this() { + assert(consumed); + } + + /++ + Schedules the job to be run at the given time. + +/ + void at(DateTime when, immutable TimeZone timezone = UTC()) { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto st = SysTime(when, timezone); + auto jobId = conn.scheduleJob(1, cast(int) st.toUnixTime(), thisExePath, func, args); + } + + /++ + Schedules the job to run at least after the specified delay. + +/ + void delay(Duration delay) { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto jobId = conn.scheduleJob(0, cast(int) delay.total!"seconds", thisExePath, func, args); + } + + /++ + Runs the job in the background ASAP. + + $(NOTE It may run in a background thread. Don't segfault!) + +/ + void asap() { + consumed = true; + + auto conn = ScheduledJobServerConnection.connection; + import std.file; + auto jobId = conn.scheduleJob(0, 1, thisExePath, func, args); + } + + /+ + /++ + Schedules the job to recur on the given pattern. + +/ + void recur(string spec) { + + } + +/ +} + +/++ + First step to schedule a job on the scheduled job server. + + The scheduled job needs to be a top-level function that doesn't read any + variables from outside its arguments because it may be run in a new process, + without any context existing later. + + You MUST set details on the returned object to actually do anything! ++/ +template schedule(alias fn, T...) if(is(typeof(fn) == function)) { + /// + ScheduledJobHelper schedule(T args) { + // this isn't meant to ever be called, but instead just to + // get the compiler to type check the arguments passed for us + auto sample = delegate() { + fn(args); + }; + string[] sargs; + foreach(arg; args) + sargs ~= to!string(arg); + return ScheduledJobHelper(fn.mangleof, sargs); + } + + shared static this() { + scheduledJobHandlers[fn.mangleof] = delegate(string[] sargs) { + import std.traits; + Parameters!fn args; + foreach(idx, ref arg; args) + arg = to!(typeof(arg))(sargs[idx]); + fn(args); + }; + } +} + +/// +interface ScheduledJobServer { + /// Use the [schedule] function for a higher-level interface. + int scheduleJob(int whenIs, int when, string executable, string func, string[] args); + /// + void cancelJob(int jobId); +} + +version(with_addon_servers_connections) +class ScheduledJobServerConnection : ScheduledJobServer { + mixin ImplementRpcClientInterface!(ScheduledJobServer, "/tmp/arsd_scheduled_job_server", "--timer-server"); +} + +version(with_addon_servers) +final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer { + // FIXME: we need to handle SIGCHLD in this somehow + // whenIs is 0 for relative, 1 for absolute + protected int scheduleJob(int whenIs, int when, string executable, string func, string[] args) { + auto nj = nextJobId; + nextJobId++; + + version(linux) { + import core.sys.linux.timerfd; + import core.sys.linux.epoll; + import core.sys.posix.unistd; + + + auto fd = timerfd_create(CLOCK_REALTIME, TFD_NONBLOCK | TFD_CLOEXEC); + if(fd == -1) + throw new Exception("fd timer create failed"); + + foreach(ref arg; args) + arg = arg.idup; + auto job = Job(executable.idup, func.idup, .dup(args), fd, nj); + + itimerspec value; + value.it_value.tv_sec = when; + value.it_value.tv_nsec = 0; + + value.it_interval.tv_sec = 0; + value.it_interval.tv_nsec = 0; + + if(timerfd_settime(fd, whenIs == 1 ? TFD_TIMER_ABSTIME : 0, &value, null) == -1) + throw new Exception("couldn't set fd timer"); + + auto op = allocateIoOp(fd, IoOp.Read, 16, (IoOp* op, int fd) { + jobs.remove(nj); + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, null); + close(fd); + + + spawnProcess([job.executable, "--timed-job", job.func] ~ job.args); + + return true; + }); + scope(failure) + freeIoOp(op); + + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = op; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + jobs[nj] = job; + return nj; + } else assert(0); + } + + protected void cancelJob(int jobId) { + version(linux) { + auto job = jobId in jobs; + if(job is null) + return; + + jobs.remove(jobId); + + version(linux) { + import core.sys.linux.timerfd; + import core.sys.linux.epoll; + import core.sys.posix.unistd; + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, job.timerfd, null); + close(job.timerfd); + } + } + jobs.remove(jobId); + } + + int nextJobId = 1; + static struct Job { + string executable; + string func; + string[] args; + int timerfd; + int id; + } + Job[int] jobs; + + + // event io server methods below + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!ScheduledJobServer(this, data, op.fd); + return false; + } + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) {} // stateless so irrelevant + + int epoll_fd_; + void epoll_fd(int fd) {this.epoll_fd_ = fd; } + int epoll_fd() { return epoll_fd_; } +} + +/++ + History: + Added January 6, 2019 ++/ +version(with_addon_servers_connections) +interface EventSourceServer { + /++ + sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. + + See_Also: + [sendEvent] + + Bugs: + Not implemented on Windows! + + History: + Officially stabilised on November 23, 2023 (dub v11.4). It actually worked pretty well in its original design. + +/ + public static void adoptConnection(Cgi cgi, in char[] eventUrl) { + /* + If lastEventId is missing or empty, you just get new events as they come. + + If it is set from something else, it sends all since then (that are still alive) + down the pipe immediately. + + The reason it can come from the header is that's what the standard defines for + browser reconnects. The reason it can come from a query string is just convenience + in catching up in a user-defined manner. + + The reason the header overrides the query string is if the browser tries to reconnect, + it will send the header AND the query (it reconnects to the same url), so we just + want to do the restart thing. + + Note that if you ask for "0" as the lastEventId, it will get ALL still living events. + */ + string lastEventId = cgi.lastEventId; + if(lastEventId.length == 0 && "lastEventId" in cgi.get) + lastEventId = cgi.get["lastEventId"]; + + cgi.setResponseContentType("text/event-stream"); + cgi.write(":\n", false); // to initialize the chunking and send headers before keeping the fd for later + cgi.flush(); + + cgi.closed = true; + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server", "--event-server"); + scope(exit) + closeLocalServerConnection(s); + + version(fastcgi) + throw new Exception("sending fcgi connections not supported"); + else { + auto fd = cgi.getOutputFileHandle(); + if(isInvalidHandle(fd)) + throw new Exception("bad fd from cgi!"); + + EventSourceServerImplementation.SendableEventConnection sec; + sec.populate(cgi.responseChunked, eventUrl, lastEventId); + + version(Posix) { + auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); + assert(res == sec.sizeof); + } else version(Windows) { + // FIXME + } + } + } + + /++ + Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later. + + Params: + url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request. + event = the event type string, which is used in the Javascript addEventListener API on EventSource + data = the event data. Available in JS as `event.data`. + lifetime = the amount of time to keep this event for replaying on the event server. + + Bugs: + Not implemented on Windows! + + History: + Officially stabilised on November 23, 2023 (dub v11.4). It actually worked pretty well in its original design. + +/ + public static void sendEvent(string url, string event, string data, int lifetime) { + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server", "--event-server"); + scope(exit) + closeLocalServerConnection(s); + + EventSourceServerImplementation.SendableEvent sev; + sev.populate(url, event, data, lifetime); + + version(Posix) { + auto ret = send(s, &sev, sev.sizeof, 0); + assert(ret == sev.sizeof); + } else version(Windows) { + // FIXME + } + } + + /++ + Messages sent to `url` will also be sent to anyone listening on `forwardUrl`. + + See_Also: [disconnect] + +/ + void connect(string url, string forwardUrl); + + /++ + Disconnects `forwardUrl` from `url` + + See_Also: [connect] + +/ + void disconnect(string url, string forwardUrl); +} + +/// +version(with_addon_servers) +final class EventSourceServerImplementation : EventSourceServer, EventIoServer { + + protected: + + void connect(string url, string forwardUrl) { + pipes[url] ~= forwardUrl; + } + void disconnect(string url, string forwardUrl) { + auto t = url in pipes; + if(t is null) + return; + foreach(idx, n; (*t)) + if(n == forwardUrl) { + (*t)[idx] = (*t)[$-1]; + (*t) = (*t)[0 .. $-1]; + break; + } + } + + bool handleLocalConnectionData(IoOp* op, int receivedFd) { + if(receivedFd != -1) { + //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); + + //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); + + SendableEventConnection* got = cast(SendableEventConnection*) op.usedBuffer.ptr; + + auto url = got.url.idup; + eventConnectionsByUrl[url] ~= EventConnection(receivedFd, got.responseChunked > 0 ? true : false); + + // FIXME: catch up on past messages here + } else { + auto data = op.usedBuffer; + auto event = cast(SendableEvent*) data.ptr; + + if(event.magic == 0xdeadbeef) { + handleInputEvent(event); + + if(event.url in pipes) + foreach(pipe; pipes[event.url]) { + event.url = pipe; + handleInputEvent(event); + } + } else { + dispatchRpcServer!EventSourceServer(this, data, op.fd); + } + } + return false; + } + void handleLocalConnectionClose(IoOp* op) { + fileClosed(op.fd); + } + void handleLocalConnectionComplete(IoOp* op) {} + + void wait_timeout() { + // just keeping alive + foreach(url, connections; eventConnectionsByUrl) + foreach(connection; connections) + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, "1b\r\nevent: keepalive\ndata: ok\n\n\r\n"); + else + nonBlockingWrite(this, connection.fd, "event: keepalive\ndata: ok\n\n\r\n"); + } + + void fileClosed(int fd) { + outer: foreach(url, ref connections; eventConnectionsByUrl) { + foreach(idx, conn; connections) { + if(fd == conn.fd) { + connections[idx] = connections[$-1]; + connections = connections[0 .. $ - 1]; + continue outer; + } + } + } + } + + void epoll_fd(int fd) {} + + + private: + + + struct SendableEventConnection { + ubyte responseChunked; + + int urlLength; + char[256] urlBuffer = 0; + + int lastEventIdLength; + char[32] lastEventIdBuffer = 0; + + char[] url() return { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + char[] lastEventId() return { + return lastEventIdBuffer[0 .. lastEventIdLength]; + } + void populate(bool responseChunked, in char[] url, in char[] lastEventId) + in { + assert(url.length < this.urlBuffer.length); + assert(lastEventId.length < this.lastEventIdBuffer.length); + } + do { + this.responseChunked = responseChunked ? 1 : 0; + this.urlLength = cast(int) url.length; + this.lastEventIdLength = cast(int) lastEventId.length; + + this.urlBuffer[0 .. url.length] = url[]; + this.lastEventIdBuffer[0 .. lastEventId.length] = lastEventId[]; + } + } + + struct SendableEvent { + int magic = 0xdeadbeef; + int urlLength; + char[256] urlBuffer = 0; + int typeLength; + char[32] typeBuffer = 0; + int messageLength; + char[2048 * 4] messageBuffer = 0; // this is an arbitrary limit, it needs to fit comfortably in stack (including in a fiber) and be a single send on the kernel side cuz of the impl... i think this is ok for a unix socket. + int _lifetime; + + char[] message() return { + return messageBuffer[0 .. messageLength]; + } + char[] type() return { + return typeBuffer[0 .. typeLength]; + } + char[] url() return { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + int lifetime() { + return _lifetime; + } + + /// + void populate(string url, string type, string message, int lifetime) + in { + assert(url.length < this.urlBuffer.length); + assert(type.length < this.typeBuffer.length); + assert(message.length < this.messageBuffer.length); + } + do { + this.urlLength = cast(int) url.length; + this.typeLength = cast(int) type.length; + this.messageLength = cast(int) message.length; + this._lifetime = lifetime; + + this.urlBuffer[0 .. url.length] = url[]; + this.typeBuffer[0 .. type.length] = type[]; + this.messageBuffer[0 .. message.length] = message[]; + } + } + + struct EventConnection { + int fd; + bool needsChunking; + } + + private EventConnection[][string] eventConnectionsByUrl; + private string[][string] pipes; + + private void handleInputEvent(scope SendableEvent* event) { + static int eventId; + + static struct StoredEvent { + int id; + string type; + string message; + int lifetimeRemaining; + } + + StoredEvent[][string] byUrl; + + int thisId = ++eventId; + + if(event.lifetime) + byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); + + auto connectionsPtr = event.url in eventConnectionsByUrl; + EventConnection[] connections; + if(connectionsPtr is null) + return; + else + connections = *connectionsPtr; + + char[4096] buffer; + char[] formattedMessage; + + void append(const char[] a) { + // the 6's here are to leave room for a HTTP chunk header, if it proves necessary + buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; + formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; + } + + import std.algorithm.iteration; + + if(connections.length) { + append("id: "); + append(to!string(thisId)); + append("\n"); + + append("event: "); + append(event.type); + append("\n"); + + foreach(line; event.message.splitter("\n")) { + append("data: "); + append(line); + append("\n"); + } + + append("\n"); + } + + // chunk it for HTTP! + auto len = toHex(formattedMessage.length); + buffer[4 .. 6] = "\r\n"[]; + buffer[4 - len.length .. 4] = len[]; + buffer[6 + formattedMessage.length] = '\r'; + buffer[6 + formattedMessage.length + 1] = '\n'; + + auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length +2]; + // done + + // FIXME: send back requests when needed + // FIXME: send a single ":\n" every 15 seconds to keep alive + + foreach(connection; connections) { + if(connection.needsChunking) { + nonBlockingWrite(this, connection.fd, chunkedMessage); + } else { + nonBlockingWrite(this, connection.fd, formattedMessage); + } + } + } +} + +void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoServer)) { + version(Posix) { + + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + import core.sys.posix.sys.un; + + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + + static extern(C) void sigchldhandler(int) { + int status; + import w = core.sys.posix.sys.wait; + w.wait(&status); + } + signal(SIGCHLD, &sigchldhandler); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + cloexec(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. localListenerName.length + 1] = cast(typeof(addr.sun_path[])) localListenerName[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. localListenerName.length] = cast(typeof(addr.sun_path[])) localListenerName[]; + } + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("bind " ~ to!string(errno)); + + if(listen(sock, 128) == -1) + throw new Exception("listen " ~ to!string(errno)); + + makeNonBlocking(sock); + + version(linux) { + import core.sys.linux.epoll; + auto epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if(epoll_fd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(failure) + close(epoll_fd); + } else { + import core.sys.posix.poll; + } + + version(linux) + eis.epoll_fd = epoll_fd; + + auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); + scope(exit) + freeIoOp(acceptOp); + + version(linux) { + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = acceptOp; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + epoll_event[64] events; + } else { + pollfd[] pollfds; + IoOp*[int] ioops; + pollfds ~= pollfd(sock, POLLIN); + ioops[sock] = acceptOp; + } + + import core.time : MonoTime, seconds; + + MonoTime timeout = MonoTime.currTime + 15.seconds; + + while(true) { + + // FIXME: it should actually do a timerfd that runs on any thing that hasn't been run recently + + int timeout_milliseconds = 0; // -1; // infinite + + timeout_milliseconds = cast(int) (timeout - MonoTime.currTime).total!"msecs"; + if(timeout_milliseconds < 0) + timeout_milliseconds = 0; + + //writeln("waiting for ", name); + + version(linux) { + auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + } else { + int nfds = poll(pollfds.ptr, cast(int) pollfds.length, timeout_milliseconds); + size_t lastIdx = 0; + } + + if(nfds == 0) { + eis.wait_timeout(); + timeout += 15.seconds; + } + + foreach(idx; 0 .. nfds) { + version(linux) { + auto flags = events[idx].events; + auto ioop = cast(IoOp*) events[idx].data.ptr; + } else { + IoOp* ioop; + foreach(tidx, thing; pollfds[lastIdx .. $]) { + if(thing.revents) { + ioop = ioops[thing.fd]; + lastIdx += tidx + 1; + break; + } + } + } + + //writeln(flags, " ", ioop.fd); + + void newConnection() { + // on edge triggering, it is important that we get it all + while(true) { + auto size = cast(socklen_t) addr.sizeof; + auto ns = accept(sock, cast(sockaddr*) &addr, &size); + if(ns == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + break; + } + throw new Exception("accept " ~ to!string(errno)); + } + cloexec(ns); + + makeNonBlocking(ns); + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096 * 4, &eis.handleLocalConnectionData); + niop.closeHandler = &eis.handleLocalConnectionClose; + niop.completeHandler = &eis.handleLocalConnectionComplete; + scope(failure) freeIoOp(niop); + + version(linux) { + epoll_event nev; + nev.events = EPOLLIN | EPOLLET; + nev.data.ptr = niop; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } else { + bool found = false; + foreach(ref pfd; pollfds) { + if(pfd.fd < 0) { + pfd.fd = ns; + found = true; + } + } + if(!found) + pollfds ~= pollfd(ns, POLLIN); + ioops[ns] = niop; + } + } + } + + bool newConnectionCondition() { + version(linux) + return ioop.fd == sock && (flags & EPOLLIN); + else + return pollfds[idx].fd == sock && (pollfds[idx].revents & POLLIN); + } + + if(newConnectionCondition()) { + newConnection(); + } else if(ioop.operation == IoOp.ReadSocketHandle) { + while(true) { + int in_fd; + auto got = read_fd(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, &in_fd); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) { + ioop.closeHandler(ioop); + version(linux) {} // nothing needed + else { + foreach(ref pfd; pollfds) { + if(pfd.fd == ioop.fd) + pfd.fd = -1; + } + } + } + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + ioop.handler(ioop, in_fd); + } + } else if(ioop.operation == IoOp.Read) { + while(true) { + auto got = read(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(ioop.fd) ~ " errno " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + if(ioop.handler(ioop, ioop.fd)) { + close(ioop.fd); + freeIoOp(ioop); + break; + } + } + } + + // EPOLLHUP? + } + } + } else version(Windows) { + + // set up a named pipe + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724251(v=vs.85).aspx + // https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsaduplicatesocketw + // https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeserverprocessid + + } else static assert(0); +} + + +version(with_sendfd) +// copied from the web and ported from C +// see https://stackoverflow.com/questions/2358684/can-i-share-a-file-descriptor-to-another-process-on-linux-or-are-they-local-to-t +ssize_t write_fd(int fd, void *ptr, size_t nbytes, int sendfd) { + msghdr msg; + iovec[1] iov; + + version(OSX) { + //msg.msg_accrights = cast(cattr_t) &sendfd; + //msg.msg_accrightslen = int.sizeof; + } else version(Android) { + } else { + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + cmptr = CMSG_FIRSTHDR(&msg); + cmptr.cmsg_len = CMSG_LEN(int.sizeof); + cmptr.cmsg_level = SOL_SOCKET; + cmptr.cmsg_type = SCM_RIGHTS; + *(cast(int *) CMSG_DATA(cmptr)) = sendfd; + } + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + return sendmsg(fd, &msg, 0); +} + +version(with_sendfd) +// copied from the web and ported from C +ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { + msghdr msg; + iovec[1] iov; + ssize_t n; + int newfd; + + version(OSX) { + //msg.msg_accrights = cast(cattr_t) recvfd; + //msg.msg_accrightslen = int.sizeof; + } else version(Android) { + } else { + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + } + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + if ( (n = recvmsg(fd, &msg, 0)) <= 0) + return n; + + version(OSX) { + //if(msg.msg_accrightslen != int.sizeof) + //*recvfd = -1; + } else version(Android) { + } else { + if ( (cmptr = CMSG_FIRSTHDR(&msg)) != null && + cmptr.cmsg_len == CMSG_LEN(int.sizeof)) { + if (cmptr.cmsg_level != SOL_SOCKET) + throw new Exception("control level != SOL_SOCKET"); + if (cmptr.cmsg_type != SCM_RIGHTS) + throw new Exception("control type != SCM_RIGHTS"); + *recvfd = *(cast(int *) CMSG_DATA(cmptr)); + } else + *recvfd = -1; /* descriptor was not passed */ + } + + return n; +} +/* end read_fd */ + + +/* + Event source stuff + + The api is: + + sendEvent(string url, string type, string data, int timeout = 60*10); + + attachEventListener(string url, int fd, lastId) + + + It just sends to all attached listeners, and stores it until the timeout + for replaying via lastEventId. +*/ + +/* + Session process stuff + + it stores it all. the cgi object has a session object that can grab it + + session may be done in the same process if possible, there is a version + switch to choose if you want to override. +*/ + +struct DispatcherDefinition(alias dispatchHandler, DispatcherDetails = typeof(null)) {// if(is(typeof(dispatchHandler("str", Cgi.init, void) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler; + alias handler = dispatchHandler; + string urlPrefix; + bool rejectFurther; + immutable(DispatcherDetails) details; +} + +private string urlify(string name) pure { + return beautify(name, '-', true); +} + +private string beautify(string name, char space = ' ', bool allLowerCase = false) pure { + if(name == "id") + return allLowerCase ? name : "ID"; + + char[160] buffer; + int bufferIndex = 0; + bool shouldCap = true; + bool shouldSpace; + bool lastWasCap; + foreach(idx, char ch; name) { + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + + if((ch >= 'A' && ch <= 'Z') || ch == '_') { + if(lastWasCap) { + // two caps in a row, don't change. Prolly acronym. + } else { + if(idx) + shouldSpace = true; // new word, add space + } + + lastWasCap = true; + } else { + lastWasCap = false; + } + + if(shouldSpace) { + buffer[bufferIndex++] = space; + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + shouldSpace = false; + } + if(shouldCap) { + if(ch >= 'a' && ch <= 'z') + ch -= 32; + shouldCap = false; + } + if(allLowerCase && ch >= 'A' && ch <= 'Z') + ch += 32; + buffer[bufferIndex++] = ch; + } + return buffer[0 .. bufferIndex].idup; +} + +/* +string urlFor(alias func)() { + return __traits(identifier, func); +} +*/ + +/++ + UDA: The name displayed to the user in auto-generated HTML. + + Default is `beautify(identifier)`. ++/ +struct DisplayName { + string name; +} + +/++ + UDA: The name used in the URL or web parameter. + + Default is `urlify(identifier)` for functions and `identifier` for parameters and data members. ++/ +struct UrlName { + string name; +} + +/++ + UDA: default format to respond for this method ++/ +struct DefaultFormat { string value; } + +class MissingArgumentException : Exception { + string functionName; + string argumentName; + string argumentType; + + this(string functionName, string argumentName, string argumentType, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.functionName = functionName; + this.argumentName = argumentName; + this.argumentType = argumentType; + + super("Missing Argument: " ~ this.argumentName, file, line, next); + } +} + +/++ + You can throw this from an api handler to indicate a 404 response. This is done by the presentExceptionAsHtml function in the presenter. + + History: + Added December 15, 2021 (dub v10.5) ++/ +class ResourceNotFoundException : Exception { + string resourceType; + string resourceId; + + this(string resourceType, string resourceId, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.resourceType = resourceType; + this.resourceId = resourceId; + + super("Resource not found: " ~ resourceType ~ " " ~ resourceId, file, line, next); + } + +} + +/++ + This can be attached to any constructor or function called from the cgi system. + + If it is present, the function argument can NOT be set from web params, but instead + is set to the return value of the given `func`. + + If `func` can take a parameter of type [Cgi], it will be passed the one representing + the current request. Otherwise, it must take zero arguments. + + Any params in your function of type `Cgi` are automatically assumed to take the cgi object + for the connection. Any of type [Session] (with an argument) is also assumed to come from + the cgi object. + + const arguments are also supported. ++/ +struct ifCalledFromWeb(alias func) {} + +// it only looks at query params for GET requests, the rest must be in the body for a function argument. +auto callFromCgi(alias method, T)(T dg, Cgi cgi) { + + // FIXME: any array of structs should also be settable or gettable from csv as well. + + // FIXME: think more about checkboxes and bools. + + import std.traits; + + Parameters!method params; + alias idents = ParameterIdentifierTuple!method; + alias defaults = ParameterDefaults!method; + + const(string)[] names; + const(string)[] values; + + // first, check for missing arguments and initialize to defaults if necessary + + static if(is(typeof(method) P == __parameters)) + foreach(idx, param; P) {{ + // see: mustNotBeSetFromWebParams + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[idx] = cgi; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[idx] = cgi.getSessionObject!D(); + } else { + bool populated; + foreach(uda; __traits(getAttributes, P[idx .. idx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[idx] = func(cgi); + else + params[idx] = func(); + + populated = true; + } + } + + if(!populated) { + static if(__traits(compiles, { params[idx] = param.getAutomaticallyForCgi(cgi); } )) { + params[idx] = param.getAutomaticallyForCgi(cgi); + populated = true; + } + } + + if(!populated) { + auto ident = idents[idx]; + if(cgi.requestMethod == Cgi.RequestMethod.GET) { + if(ident !in cgi.get) { + static if(is(defaults[idx] == void)) { + static if(is(param == bool)) + params[idx] = false; + else + throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof); + } else + params[idx] = defaults[idx]; + } + } else { + if(ident !in cgi.post) { + static if(is(defaults[idx] == void)) { + static if(is(param == bool)) + params[idx] = false; + else + throw new MissingArgumentException(__traits(identifier, method), ident, param.stringof); + } else + params[idx] = defaults[idx]; + } + } + } + } + }} + + // second, parse the arguments in order to build up arrays, etc. + + static bool setVariable(T)(string name, string paramName, T* what, string value) { + static if(is(T == struct)) { + if(name == paramName) { + *what = T.init; + return true; + } else { + // could be a child. gonna allow either obj.field OR obj[field] + + string afterName; + + if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + afterName = name[idx + 1 .. $]; + + name = name[0 .. paramName.length]; + + paramName = insideBrackets; + + } else if(name[paramName.length] == '.') { + paramName = name[paramName.length + 1 .. $]; + name = paramName; + int p = 0; + foreach(ch; paramName) { + if(ch == '.' || ch == '[') + break; + p++; + } + + afterName = paramName[p .. $]; + paramName = paramName[0 .. p]; + } else { + return false; + } + + if(paramName.length) + // set the child member + switch(paramName) { + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + // data member! + case memberName: + return setVariable(name ~ afterName, paramName, &(__traits(getMember, *what, memberName)), value); + } + default: + // ok, not a member + } + } + + return false; + } else static if(is(T == enum)) { + *what = to!T(value); + return true; + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + *what = to!T(value); + return true; + } else static if(is(T == bool)) { + *what = value == "1" || value == "yes" || value == "t" || value == "true" || value == "on"; + return true; + } else static if(is(T == K[], K)) { + K tmp; + if(name == paramName) { + // direct - set and append + if(setVariable(name, paramName, &tmp, value)) { + (*what) ~= tmp; + return true; + } else { + return false; + } + } else { + // child, append to last element + // FIXME: what about range violations??? + auto ptr = &(*what)[(*what).length - 1]; + return setVariable(name, paramName, ptr, value); + + } + } else static if(is(T == V[K], K, V)) { + // assoc array, name[key] is valid + if(name == paramName) { + // no action necessary + return true; + } else if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + auto afterName = name[idx + 1 .. $]; + + auto k = to!K(insideBrackets); + V v; + if(auto ptr = k in *what) + v = *ptr; + + name = name[0 .. paramName.length]; + //writeln(name, afterName, " ", paramName); + + auto ret = setVariable(name ~ afterName, paramName, &v, value); + if(ret) { + (*what)[k] = v; + return true; + } + } + + return false; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + //return false; + } + + void setArgument(string name, string value) { + int p; + foreach(ch; name) { + if(ch == '.' || ch == '[') + break; + p++; + } + + auto paramName = name[0 .. p]; + + sw: switch(paramName) { + static if(is(typeof(method) P == __parameters)) + foreach(idx, param; P) { + static if(mustNotBeSetFromWebParams!(P[idx], __traits(getAttributes, P[idx .. idx + 1]))) { + // cannot be set from the outside + } else { + case idents[idx]: + static if(is(param == Cgi.UploadedFile)) { + params[idx] = cgi.files[name]; + } else static if(is(param : const Cgi.UploadedFile[])) { + (cast() params[idx]) = cgi.filesArray[name]; + } else { + setVariable(name, paramName, ¶ms[idx], value); + } + break sw; + } + } + default: + // ignore; not relevant argument + } + } + + if(cgi.requestMethod == Cgi.RequestMethod.GET) { + names = cgi.allGetNamesInOrder; + values = cgi.allGetValuesInOrder; + } else { + names = cgi.allPostNamesInOrder; + values = cgi.allPostValuesInOrder; + } + + foreach(idx, name; names) { + setArgument(name, values[idx]); + } + + static if(is(ReturnType!method == void)) { + typeof(null) ret; + dg(params); + } else { + auto ret = dg(params); + } + + // FIXME: format return values + // options are: json, html, csv. + // also may need to wrap in envelope format: none, html, or json. + return ret; +} + +private bool mustNotBeSetFromWebParams(T, attrs...)() { + static if(is(T : const(Cgi))) { + return true; + } else static if(is(T : const(Session!D), D)) { + return true; + } else static if(__traits(compiles, T.getAutomaticallyForCgi(Cgi.init))) { + return true; + } else { + foreach(uda; attrs) + static if(is(uda == ifCalledFromWeb!func, alias func)) + return true; + return false; + } +} + +private bool hasIfCalledFromWeb(attrs...)() { + foreach(uda; attrs) + static if(is(uda == ifCalledFromWeb!func, alias func)) + return true; + return false; +} + +/++ + Implies POST path for the thing itself, then GET will get the automatic form. + + The given customizer, if present, will be called as a filter on the Form object. + + History: + Added December 27, 2020 ++/ +template AutomaticForm(alias customizer) { } + +/++ + This is meant to be returned by a function that takes a form POST submission. You + want to set the url of the new resource it created, which is set as the http + Location header for a "201 Created" result, and you can also set a separate + destination for browser users, which it sets via a "Refresh" header. + + The `resourceRepresentation` should generally be the thing you just created, and + it will be the body of the http response when formatted through the presenter. + The exact thing is up to you - it could just return an id, or the whole object, or + perhaps a partial object. + + Examples: + --- + class Test : WebObject { + @(Cgi.RequestMethod.POST) + CreatedResource!int makeThing(string value) { + return CreatedResource!int(value.to!int, "/resources/id"); + } + } + --- + + History: + Added December 18, 2021 ++/ +struct CreatedResource(T) { + static if(!is(T == void)) + T resourceRepresentation; + string resourceUrl; + string refreshUrl; +} + +/+ +/++ + This can be attached as a UDA to a handler to add a http Refresh header on a + successful run. (It will not be attached if the function throws an exception.) + This will refresh the browser the given number of seconds after the page loads, + to the url returned by `urlFunc`, which can be either a static function or a + member method of the current handler object. + + You might use this for a POST handler that is normally used from ajax, but you + want it to degrade gracefully to a temporarily flashed message before reloading + the main page. + + History: + Added December 18, 2021 ++/ +struct Refresh(alias urlFunc) { + int waitInSeconds; + + string url() { + static if(__traits(isStaticFunction, urlFunc)) + return urlFunc(); + else static if(is(urlFunc : string)) + return urlFunc; + } +} ++/ + +/+ +/++ + Sets a filter to be run before + + A before function can do validations of params and log and stop the function from running. ++/ +template Before(alias b) {} +template After(alias b) {} ++/ + +/+ + Argument conversions: for the most part, it is to!Thing(string). + + But arrays and structs are a bit different. Arrays come from the cgi array. Thus + they are passed + + arr=foo&arr=bar <-- notice the same name. + + Structs are first declared with an empty thing, then have their members set individually, + with dot notation. The members are not required, just the initial declaration. + + struct Foo { + int a; + string b; + } + void test(Foo foo){} + + foo&foo.a=5&foo.b=str <-- the first foo declares the arg, the others set the members + + Arrays of structs use this declaration. + + void test(Foo[] foo) {} + + foo&foo.a=5&foo.b=bar&foo&foo.a=9 + + You can use a hidden input field in HTML forms to achieve this. The value of the naked name + declaration is ignored. + + Mind that order matters! The declaration MUST come first in the string. + + Arrays of struct members follow this rule recursively. + + struct Foo { + int[] a; + } + + foo&foo.a=1&foo.a=2&foo&foo.a=1 + + + Associative arrays are formatted with brackets, after a declaration, like structs: + + foo&foo[key]=value&foo[other_key]=value + + + Note: for maximum compatibility with outside code, keep your types simple. Some libraries + do not support the strict ordering requirements to work with these struct protocols. + + FIXME: also perhaps accept application/json to better work with outside trash. + + + Return values are also auto-formatted according to user-requested type: + for json, it loops over and converts. + for html, basic types are strings. Arrays are
    . Structs are
    . Arrays of structs are tables! ++/ + +/++ + A web presenter is responsible for rendering things to HTML to be usable + in a web browser. + + They are passed as template arguments to the base classes of [WebObject] + + Responsible for displaying stuff as HTML. You can put this into your own aggregate + and override it. Use forwarding and specialization to customize it. + + When you inherit from it, pass your own class as the CRTP argument. This lets the base + class templates and your overridden templates work with each other. + + --- + class MyPresenter : WebPresenter!(MyPresenter) { + @Override + void presentSuccessfulReturnAsHtml(T : CustomType)(Cgi cgi, T ret, typeof(null) meta) { + // present the CustomType + } + @Override + void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret, typeof(null) meta) { + // handle everything else via the super class, which will call + // back to your class when appropriate + super.presentSuccessfulReturnAsHtml(cgi, ret); + } + } + --- + + The meta argument in there can be overridden by your own facility. + ++/ +class WebPresenter(CRTP) { + + /// A UDA version of the built-in `override`, to be used for static template polymorphism + /// If you override a plain method, use `override`. If a template, use `@Override`. + enum Override; + + string script() { + return ` + `; + } + + string style() { + return ` + :root { + --mild-border: #ccc; + --middle-border: #999; + --accent-color: #f2f2f2; + --sidebar-color: #fefefe; + } + ` ~ genericFormStyling() ~ genericSiteStyling(); + } + + string genericFormStyling() { + return +q"css + table.automatic-data-display { + border-collapse: collapse; + border: solid 1px var(--mild-border); + } + + table.automatic-data-display td { + vertical-align: top; + border: solid 1px var(--mild-border); + padding: 2px 4px; + } + + table.automatic-data-display th { + border: solid 1px var(--mild-border); + border-bottom: solid 1px var(--middle-border); + padding: 2px 4px; + } + + ol.automatic-data-display { + margin: 0px; + list-style-position: inside; + padding: 0px; + } + + dl.automatic-data-display { + + } + + .automatic-form { + max-width: 600px; + } + + .form-field { + margin: 0.5em; + padding-left: 0.5em; + } + + .label-text { + display: block; + font-weight: bold; + margin-left: -0.5em; + } + + .submit-button-holder { + padding-left: 2em; + } + + .add-array-button { + + } +css"; + } + + string genericSiteStyling() { + return +q"css + * { box-sizing: border-box; } + html, body { margin: 0px; } + body { + font-family: sans-serif; + } + header { + background: var(--accent-color); + height: 64px; + } + footer { + background: var(--accent-color); + height: 64px; + } + #site-container { + display: flex; + flex-wrap: wrap; + } + main { + flex: 1 1 auto; + order: 2; + min-height: calc(100vh - 64px - 64px); + min-width: 80ch; + padding: 4px; + padding-left: 1em; + } + #sidebar { + flex: 0 0 16em; + order: 1; + background: var(--sidebar-color); + } +css"; + } + + import arsd.dom; + Element htmlContainer() { + auto document = new Document(q"html + + + + + + D Application + + + + +
    +
    +
    + +
    +
    + + + +html", true, true); + + return document.requireSelector("main"); + } + + /// Renders a response as an HTTP error with associated html body + void renderBasicError(Cgi cgi, int httpErrorCode) { + cgi.setResponseStatus(getHttpCodeText(httpErrorCode)); + auto c = htmlContainer(); + c.innerText = getHttpCodeText(httpErrorCode); + cgi.setResponseContentType("text/html; charset=utf-8"); + cgi.write(c.parentDocument.toString(), true); + } + + template methodMeta(alias method) { + enum methodMeta = null; + } + + void presentSuccessfulReturn(T, Meta)(Cgi cgi, T ret, Meta meta, string format) { + switch(format) { + case "html": + (cast(CRTP) this).presentSuccessfulReturnAsHtml(cgi, ret, meta); + break; + case "json": + import arsd.jsvar; + static if(is(typeof(ret) == MultipleResponses!Types, Types...)) { + var json; + foreach(index, type; Types) { + if(ret.contains == index) + json = ret.payload[index]; + } + } else { + var json = ret; + } + var envelope = json; // var.emptyObject; + /* + envelope.success = true; + envelope.result = json; + envelope.error = null; + */ + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + break; + default: + cgi.setResponseStatus("406 Not Acceptable"); // not exactly but sort of. + } + } + + /// typeof(null) (which is also used to represent functions returning `void`) do nothing + /// in the default presenter - allowing the function to have full low-level control over the + /// response. + void presentSuccessfulReturn(T : typeof(null), Meta)(Cgi cgi, T ret, Meta meta, string format) { + // nothing intentionally! + } + + /// Redirections are forwarded to [Cgi.setResponseLocation] + void presentSuccessfulReturn(T : Redirection, Meta)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setResponseLocation(ret.to, true, getHttpCodeText(ret.code)); + } + + /// [CreatedResource]s send code 201 and will set the given urls, then present the given representation. + void presentSuccessfulReturn(T : CreatedResource!R, Meta, R)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setResponseStatus(getHttpCodeText(201)); + if(ret.resourceUrl.length) + cgi.header("Location: " ~ ret.resourceUrl); + if(ret.refreshUrl.length) + cgi.header("Refresh: 0;" ~ ret.refreshUrl); + static if(!is(R == void)) + presentSuccessfulReturn(cgi, ret.resourceRepresentation, meta, format); + } + + /// Multiple responses deconstruct the algebraic type and forward to the appropriate handler at runtime + void presentSuccessfulReturn(T : MultipleResponses!Types, Meta, Types...)(Cgi cgi, T ret, Meta meta, string format) { + bool outputted = false; + foreach(index, type; Types) { + if(ret.contains == index) { + assert(!outputted); + outputted = true; + (cast(CRTP) this).presentSuccessfulReturn(cgi, ret.payload[index], meta, format); + } + } + if(!outputted) + assert(0); + } + + /++ + An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort if the filename member is non-null of the FileResource interface. + +/ + void presentSuccessfulReturn(T : FileResource, Meta)(Cgi cgi, T ret, Meta meta, string format) { + cgi.setCache(true); // not necessarily true but meh + if(auto fn = ret.filename()) { + cgi.header("Content-Disposition: attachment; filename="~fn~";"); + } + cgi.setResponseContentType(ret.contentType); + cgi.write(ret.getData(), true); + } + + /// And the default handler for HTML will call [formatReturnValueAsHtml] and place it inside the [htmlContainer]. + void presentSuccessfulReturnAsHtml(T)(Cgi cgi, T ret, typeof(null) meta) { + auto container = this.htmlContainer(); + container.appendChild(formatReturnValueAsHtml(ret)); + cgi.write(container.parentDocument.toString(), true); + } + + /++ + + History: + Added January 23, 2023 (dub v11.0) + +/ + void presentExceptionalReturn(Meta)(Cgi cgi, Throwable t, Meta meta, string format) { + switch(format) { + case "html": + presentExceptionAsHtml(cgi, t, meta); + break; + case "json": + presentExceptionAsJsonImpl(cgi, t); + break; + default: + } + } + + private void presentExceptionAsJsonImpl()(Cgi cgi, Throwable t) { + cgi.setResponseStatus("500 Internal Server Error"); + cgi.setResponseContentType("application/json"); + import arsd.jsvar; + var v = var.emptyObject; + v.type = typeid(t).toString; + v.msg = t.msg; + v.fullString = t.toString(); + cgi.write(v.toJson(), true); + } + + + /++ + If you override this, you will need to cast the exception type `t` dynamically, + but can then use the template arguments here to refer back to the function. + + `func` is an alias to the method itself, and `dg` is a callable delegate to the same + method on the live object. You could, in theory, change arguments and retry, but I + provide that information mostly with the expectation that you will use them to make + useful forms or richer error messages for the user. + + History: + BREAKING CHANGE on January 23, 2023 (v11.0 ): it previously took an `alias func` and `T dg` to call the function again. + I removed this in favor of a `Meta` param. + + Before: `void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg)` + + After: `void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta)` + + If you used the func for something, move that something into your `methodMeta` template. + + What is the benefit of this change? Somewhat smaller executables and faster builds thanks to more reused functions, together with + enabling an easier implementation of [presentExceptionalReturn]. + +/ + void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta) { + Form af; + /+ + foreach(attr; __traits(getAttributes, func)) { + static if(__traits(isSame, attr, AutomaticForm)) { + af = createAutomaticFormForFunction!(func)(dg); + } + } + +/ + presentExceptionAsHtmlImpl(cgi, t, af); + } + + void presentExceptionAsHtmlImpl(Cgi cgi, Throwable t, Form automaticForm) { + if(auto e = cast(ResourceNotFoundException) t) { + auto container = this.htmlContainer(); + + container.addChild("p", e.msg); + + if(!cgi.outputtedResponseData) + cgi.setResponseStatus("404 Not Found"); + cgi.write(container.parentDocument.toString(), true); + } else if(auto mae = cast(MissingArgumentException) t) { + if(automaticForm is null) + goto generic; + auto container = this.htmlContainer(); + if(cgi.requestMethod == Cgi.RequestMethod.POST) + container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing")); + container.appendChild(automaticForm); + + cgi.write(container.parentDocument.toString(), true); + } else { + generic: + auto container = this.htmlContainer(); + + // import std.stdio; writeln(t.toString()); + + container.appendChild(exceptionToElement(t)); + + container.addChild("h4", "GET"); + foreach(k, v; cgi.get) { + auto deets = container.addChild("details"); + deets.addChild("summary", k); + deets.addChild("div", v); + } + + container.addChild("h4", "POST"); + foreach(k, v; cgi.post) { + auto deets = container.addChild("details"); + deets.addChild("summary", k); + deets.addChild("div", v); + } + + + if(!cgi.outputtedResponseData) + cgi.setResponseStatus("500 Internal Server Error"); + cgi.write(container.parentDocument.toString(), true); + } + } + + Element exceptionToElement(Throwable t) { + auto div = Element.make("div"); + div.addClass("exception-display"); + + div.addChild("p", t.msg); + div.addChild("p", "Inner code origin: " ~ typeid(t).name ~ "@" ~ t.file ~ ":" ~ to!string(t.line)); + + auto pre = div.addChild("pre"); + string s; + s = t.toString(); + Element currentBox; + bool on = false; + foreach(line; s.splitLines) { + if(!on && line.startsWith("-----")) + on = true; + if(!on) continue; + if(line.indexOf("arsd/") != -1) { + if(currentBox is null) { + currentBox = pre.addChild("details"); + currentBox.addChild("summary", "Framework code"); + } + currentBox.addChild("span", line ~ "\n"); + } else { + pre.addChild("span", line ~ "\n"); + currentBox = null; + } + } + + return div; + } + + /++ + Returns an element for a particular type + +/ + Element elementFor(T)(string displayName, string name, Element function() udaSuggestion) { + import std.traits; + + auto div = Element.make("div"); + div.addClass("form-field"); + + static if(is(T : const Cgi.UploadedFile)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + i.attrs.type = "file"; + i.attrs.multiple = "multiple"; + } else static if(is(T == Cgi.UploadedFile)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + i.attrs.type = "file"; + } else static if(is(T == enum)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("select", name); + i.attrs.name = name; + + foreach(memberName; __traits(allMembers, T)) + i.addChild("option", memberName); + + } else static if(is(T == struct)) { + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto fieldset = div.addChild("fieldset"); + fieldset.addChild("legend", beautify(T.stringof)); // FIXME + fieldset.addChild("input", name); + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName, null /* FIXME: pull off the UDA */)); + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + Element i; + if(udaSuggestion) { + i = udaSuggestion(); + lbl.appendChild(i); + } else { + i = lbl.addChild("input", name); + } + i.attrs.name = name; + static if(isSomeString!T) + i.attrs.type = "text"; + else + i.attrs.type = "number"; + if(i.tagName == "textarea") + i.textContent = to!string(T.init); + else + i.attrs.value = to!string(T.init); + } else static if(is(T == bool)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.type = "checkbox"; + i.attrs.value = "true"; + i.attrs.name = name; + } else static if(is(T == K[], K)) { + auto templ = div.addChild("template"); + templ.appendChild(elementFor!(K)(null, name, null /* uda??*/)); + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto btn = div.addChild("button"); + btn.addClass("add-array-button"); + btn.attrs.type = "button"; + btn.innerText = "Add"; + btn.attrs.onclick = q{ + var a = document.importNode(this.parentNode.firstChild.content, true); + this.parentNode.insertBefore(a, this); + }; + } else static if(is(T == V[K], K, V)) { + div.innerText = "assoc array not implemented for automatic form at this time"; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + + return div; + } + + /// creates a form for gathering the function's arguments + Form createAutomaticFormForFunction(alias method, T)(T dg) { + + auto form = cast(Form) Element.make("form"); + + form.method = "POST"; // FIXME + + form.addClass("automatic-form"); + + string formDisplayName = beautify(__traits(identifier, method)); + foreach(attr; __traits(getAttributes, method)) + static if(is(typeof(attr) == DisplayName)) + formDisplayName = attr.name; + form.addChild("h3", formDisplayName); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static if(is(typeof(method) P == __parameters)) + foreach(idx, _; P) {{ + + alias param = P[idx .. idx + 1]; + + static if(!mustNotBeSetFromWebParams!(param[0], __traits(getAttributes, param))) { + string displayName = beautify(__traits(identifier, param)); + Element function() element; + foreach(attr; __traits(getAttributes, param)) { + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + else static if(is(typeof(attr) : typeof(element))) { + element = attr; + } + } + auto i = form.appendChild(elementFor!(param)(displayName, __traits(identifier, param), element)); + if(i.querySelector("input[type=file]") !is null) + form.setAttribute("enctype", "multipart/form-data"); + } + }} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; + } + + /// creates a form for gathering object members (for the REST object thing right now) + Form createAutomaticFormForObject(T)(T obj) { + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, T))); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + foreach(idx, memberName; __traits(derivedMembers, T)) {{ + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + string displayName = beautify(memberName); + Element function() element; + foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + else static if(is(typeof(attr) : typeof(element))) + element = attr; + form.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(displayName, memberName, element)); + + form.setValue(memberName, to!string(__traits(getMember, obj, memberName))); + }}} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; + } + + /// + Element formatReturnValueAsHtml(T)(T t) { + import std.traits; + + static if(is(T == typeof(null))) { + return Element.make("span"); + } else static if(is(T : Element)) { + return t; + } else static if(is(T == MultipleResponses!Types, Types...)) { + foreach(index, type; Types) { + if(t.contains == index) + return formatReturnValueAsHtml(t.payload[index]); + } + assert(0); + } else static if(is(T == Paginated!E, E)) { + auto e = Element.make("div").addClass("paginated-result"); + e.appendChild(formatReturnValueAsHtml(t.items)); + if(t.nextPageUrl.length) + e.appendChild(Element.make("a", "Next Page", t.nextPageUrl)); + return e; + } else static if(isIntegral!T || isSomeString!T || isFloatingPoint!T) { + return Element.make("span", to!string(t), "automatic-data-display"); + } else static if(is(T == V[K], K, V)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display associative-array"); + foreach(k, v; t) { + dl.addChild("dt", to!string(k)); + dl.addChild("dd", formatReturnValueAsHtml(v)); + } + return dl; + } else static if(is(T == struct)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display struct"); + + foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + dl.addChild("dt", beautify(memberName)); + dl.addChild("dd", formatReturnValueAsHtml(__traits(getMember, t, memberName))); + } + + return dl; + } else static if(is(T == bool)) { + return Element.make("span", t ? "true" : "false", "automatic-data-display"); + } else static if(is(T == E[], E)) { + static if(is(E : RestObject!Proxy, Proxy)) { + // treat RestObject similar to struct + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + static if(memberName == "id") { + string val = to!string(__traits(getMember, l, memberName)); + tr.addChild("td", Element.make("a", val, E.stringof.toLower ~ "s/" ~ val)); // FIXME + } else { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + } + + return table; + } else static if(is(E == struct)) { + // an array of structs is kinda special in that I like + // having those formatted as tables. + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + + return table; + } else { + // otherwise, I will just make a list. + auto ol = Element.make("ol"); + ol.addClass("automatic-data-display"); + foreach(e; t) + ol.addChild("li", formatReturnValueAsHtml(e)); + return ol; + } + } else static if(is(T : Object)) { + static if(is(typeof(t.toHtml()))) // FIXME: maybe i will make this an interface + return Element.make("div", t.toHtml()); + else + return Element.make("div", t.toString()); + } else static assert(0, "bad return value for cgi call " ~ T.stringof); + + assert(0); + } + +} + +/++ + The base class for the [dispatcher] function and object support. ++/ +class WebObject { + //protected Cgi cgi; + + protected void initialize(Cgi cgi) { + //this.cgi = cgi; + } +} + +/++ + Can return one of the given types, decided at runtime. The syntax + is to declare all the possible types in the return value, then you + can `return typeof(return)(...value...)` to construct it. + + It has an auto-generated constructor for each value it can hold. + + --- + MultipleResponses!(Redirection, string) getData(int how) { + if(how & 1) + return typeof(return)(Redirection("http://dpldocs.info/")); + else + return typeof(return)("hi there!"); + } + --- + + If you have lots of returns, you could, inside the function, `alias r = typeof(return);` to shorten it a little. ++/ +struct MultipleResponses(T...) { + private size_t contains; + private union { + private T payload; + } + + static foreach(index, type; T) + public this(type t) { + contains = index; + payload[index] = t; + } + + /++ + This is primarily for testing. It is your way of getting to the response. + + Let's say you wanted to test that one holding a Redirection and a string actually + holds a string, by name of "test": + + --- + auto valueToTest = your_test_function(); + + valueToTest.visit( + (Redirection r) { assert(0); }, // got a redirection instead of a string, fail the test + (string s) { assert(s == "test"); } // right value, go ahead and test it. + ); + --- + + History: + Was horribly broken until June 16, 2022. Ironically, I wrote it for tests but never actually tested it. + It tried to use alias lambdas before, but runtime delegates work much better so I changed it. + +/ + void visit(Handlers...)(Handlers handlers) { + template findHandler(type, int count, HandlersToCheck...) { + static if(HandlersToCheck.length == 0) + enum findHandler = -1; + else { + static if(is(typeof(HandlersToCheck[0].init(type.init)))) + enum findHandler = count; + else + enum findHandler = findHandler!(type, count + 1, HandlersToCheck[1 .. $]); + } + } + foreach(index, type; T) { + enum handlerIndex = findHandler!(type, 0, Handlers); + static if(handlerIndex == -1) + static assert(0, "Type " ~ type.stringof ~ " was not handled by visitor"); + else { + if(index == this.contains) + handlers[handlerIndex](this.payload[index]); + } + } + } + + /+ + auto toArsdJsvar()() { + import arsd.jsvar; + return var(null); + } + +/ +} + +// FIXME: implement this somewhere maybe +struct RawResponse { + int code; + string[] headers; + const(ubyte)[] responseBody; +} + +/++ + You can return this from [WebObject] subclasses for redirections. + + (though note the static types means that class must ALWAYS redirect if + you return this directly. You might want to return [MultipleResponses] if it + can be conditional) ++/ +struct Redirection { + string to; /// The URL to redirect to. + int code = 303; /// The HTTP code to return. +} + +/++ + Serves a class' methods, as a kind of low-state RPC over the web. To be used with [dispatcher]. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar] unless you have overriden + the presenter in the dispatcher. + + FIXME: explain this better + + You can overload functions to a limited extent: you can provide a zero-arg and non-zero-arg function, + and non-zero-arg functions can filter via UDAs for various http methods. Do not attempt other overloads, + the runtime result of that is undefined. + + A method is assumed to allow any http method unless it lists some in UDAs, in which case it is limited to only those. + (this might change, like maybe i will use pure as an indicator GET is ok. idk.) + + $(WARNING + --- + // legal in D, undefined runtime behavior with cgi.d, it may call either method + // even if you put different URL udas on it, the current code ignores them. + void foo(int a) {} + void foo(string a) {} + --- + ) + + See_Also: [serveRestObject], [serveStaticFile] ++/ +auto serveApi(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] == '/'); + return serveApiInternal!T(urlPrefix); +} + +private string nextPieceFromSlash(ref string remainingUrl) { + if(remainingUrl.length == 0) + return remainingUrl; + int slash = 0; + while(slash < remainingUrl.length && remainingUrl[slash] != '/') // && remainingUrl[slash] != '.') + slash++; + + // I am specifically passing `null` to differentiate it vs empty string + // so in your ctor, `items` means new T(null) and `items/` means new T("") + auto ident = remainingUrl.length == 0 ? null : remainingUrl[0 .. slash]; + // so if it is the last item, the dot can be used to load an alternative view + // otherwise tho the dot is considered part of the identifier + // FIXME + + // again notice "" vs null here! + if(slash == remainingUrl.length) + remainingUrl = null; + else + remainingUrl = remainingUrl[slash + 1 .. $]; + + return ident; +} + +/++ + UDA used to indicate to the [dispatcher] that a trailing slash should always be added to or removed from the url. It will do it as a redirect header as-needed. ++/ +enum AddTrailingSlash; +/// ditto +enum RemoveTrailingSlash; + +private auto serveApiInternal(T)(string urlPrefix) { + + import arsd.dom; + import arsd.jsvar; + + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, immutable void* details) { + string remainingUrl = cgi.pathInfo[urlPrefix.length .. $]; + + try { + // see duplicated code below by searching subresource_ctor + // also see mustNotBeSetFromWebParams + + static if(is(typeof(T.__ctor) P == __parameters)) { + P params; + + foreach(pidx, param; P) { + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi.getSessionObject!D(); + + } else { + static if(hasIfCalledFromWeb!(__traits(getAttributes, P[pidx .. pidx + 1]))) { + foreach(uda; __traits(getAttributes, P[pidx .. pidx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[pidx] = func(cgi); + else + params[pidx] = func(); + } + } + } else { + + static if(__traits(compiles, { params[pidx] = param.getAutomaticallyForCgi(cgi); } )) { + params[pidx] = param.getAutomaticallyForCgi(cgi); + } else static if(is(param == string)) { + auto ident = nextPieceFromSlash(remainingUrl); + params[pidx] = ident; + } else static assert(0, "illegal type for subresource " ~ param.stringof); + } + } + } + + auto obj = new T(params); + } else { + auto obj = new T(); + } + + return internalHandlerWithObject(obj, remainingUrl, cgi, presenter); + } catch(Throwable t) { + switch(cgi.request("format", "html")) { + case "html": + static void dummy() {} + presenter.presentExceptionAsHtml(cgi, t, null); + return true; + case "json": + var envelope = var.emptyObject; + envelope.success = false; + envelope.result = null; + envelope.error = t.toString(); + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + return true; + default: + throw t; + // return true; + } + // return true; + } + + assert(0); + } + + static bool internalHandlerWithObject(T, Presenter)(T obj, string remainingUrl, Cgi cgi, Presenter presenter) { + + obj.initialize(cgi); + + /+ + Overload rules: + Any unique combination of HTTP verb and url path can be dispatched to function overloads + statically. + + Moreover, some args vs no args can be overloaded dynamically. + +/ + + auto methodNameFromUrl = nextPieceFromSlash(remainingUrl); + /+ + auto orig = remainingUrl; + assert(0, + (orig is null ? "__null" : orig) + ~ " .. " ~ + (methodNameFromUrl is null ? "__null" : methodNameFromUrl)); + +/ + + if(methodNameFromUrl is null) + methodNameFromUrl = "__null"; + + string hack = to!string(cgi.requestMethod) ~ " " ~ methodNameFromUrl; + + if(remainingUrl.length) + hack ~= "/"; + + switch(hack) { + foreach(methodName; __traits(derivedMembers, T)) + static if(methodName != "__ctor") + foreach(idx, overload; __traits(getOverloads, T, methodName)) { + static if(is(typeof(overload) P == __parameters)) + static if(is(typeof(overload) R == return)) + static if(__traits(getProtection, overload) == "public" || __traits(getProtection, overload) == "export") + { + static foreach(urlNameForMethod; urlNamesForMethod!(overload, urlify(methodName))) + case urlNameForMethod: + + static if(is(R : WebObject)) { + // if it returns a WebObject, it is considered a subresource. That means the url is dispatched like the ctor above. + + // the only argument it is allowed to take, outside of cgi, session, and set up thingies, is a single string + + // subresource_ctor + // also see mustNotBeSetFromWebParams + + P params; + + string ident; + + foreach(pidx, param; P) { + static if(is(param : Cgi)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi; + } else static if(is(param == typeof(presenter))) { + cast() param[pidx] = presenter; + } else static if(is(param == Session!D, D)) { + static assert(!is(param == immutable)); + cast() params[pidx] = cgi.getSessionObject!D(); + } else { + static if(hasIfCalledFromWeb!(__traits(getAttributes, P[pidx .. pidx + 1]))) { + foreach(uda; __traits(getAttributes, P[pidx .. pidx + 1])) { + static if(is(uda == ifCalledFromWeb!func, alias func)) { + static if(is(typeof(func(cgi)))) + params[pidx] = func(cgi); + else + params[pidx] = func(); + } + } + } else { + + static if(__traits(compiles, { params[pidx] = param.getAutomaticallyForCgi(cgi); } )) { + params[pidx] = param.getAutomaticallyForCgi(cgi); + } else static if(is(param == string)) { + ident = nextPieceFromSlash(remainingUrl); + if(ident is null) { + // trailing slash mandated on subresources + cgi.setResponseLocation(cgi.pathInfo ~ "/"); + return true; + } else { + params[pidx] = ident; + } + } else static assert(0, "illegal type for subresource " ~ param.stringof); + } + } + } + + auto nobj = (__traits(getOverloads, obj, methodName)[idx])(ident); + return internalHandlerWithObject!(typeof(nobj), Presenter)(nobj, remainingUrl, cgi, presenter); + } else { + // 404 it if any url left - not a subresource means we don't get to play with that! + if(remainingUrl.length) + return false; + + bool automaticForm; + + foreach(attr; __traits(getAttributes, overload)) + static if(is(attr == AddTrailingSlash)) { + if(remainingUrl is null) { + cgi.setResponseLocation(cgi.pathInfo ~ "/"); + return true; + } + } else static if(is(attr == RemoveTrailingSlash)) { + if(remainingUrl !is null) { + cgi.setResponseLocation(cgi.pathInfo[0 .. lastIndexOf(cgi.pathInfo, "/")]); + return true; + } + + } else static if(__traits(isSame, AutomaticForm, attr)) { + automaticForm = true; + } + + /+ + int zeroArgOverload = -1; + int overloadCount = cast(int) __traits(getOverloads, T, methodName).length; + bool calledWithZeroArgs = true; + foreach(k, v; cgi.get) + if(k != "format") { + calledWithZeroArgs = false; + break; + } + foreach(k, v; cgi.post) + if(k != "format") { + calledWithZeroArgs = false; + break; + } + + // first, we need to go through and see if there is an empty one, since that + // changes inside. But otherwise, all the stuff I care about can be done via + // simple looping (other improper overloads might be flagged for runtime semantic check) + // + // an argument of type Cgi is ignored for these purposes + static foreach(idx, overload; __traits(getOverloads, T, methodName)) {{ + static if(is(typeof(overload) P == __parameters)) + static if(P.length == 0) + zeroArgOverload = cast(int) idx; + else static if(P.length == 1 && is(P[0] : Cgi)) + zeroArgOverload = cast(int) idx; + }} + // FIXME: static assert if there are multiple non-zero-arg overloads usable with a single http method. + bool overloadHasBeenCalled = false; + static foreach(idx, overload; __traits(getOverloads, T, methodName)) {{ + bool callFunction = true; + // there is a zero arg overload and this is NOT it, and we have zero args - don't call this + if(overloadCount > 1 && zeroArgOverload != -1 && idx != zeroArgOverload && calledWithZeroArgs) + callFunction = false; + // if this is the zero-arg overload, obviously it cannot be called if we got any args. + if(overloadCount > 1 && idx == zeroArgOverload && !calledWithZeroArgs) + callFunction = false; + + // FIXME: so if you just add ?foo it will give the error below even when. this might not be a great idea. + + bool hadAnyMethodRestrictions = false; + bool foundAcceptableMethod = false; + foreach(attr; __traits(getAttributes, overload)) { + static if(is(typeof(attr) == Cgi.RequestMethod)) { + hadAnyMethodRestrictions = true; + if(attr == cgi.requestMethod) + foundAcceptableMethod = true; + } + } + + if(hadAnyMethodRestrictions && !foundAcceptableMethod) + callFunction = false; + + /+ + The overloads we really want to allow are the sane ones + from the web perspective. Which is likely on HTTP verbs, + for the most part, but might also be potentially based on + some args vs zero args, or on argument names. Can't really + do argument types very reliable through the web though; those + should probably be different URLs. + + Even names I feel is better done inside the function, so I'm not + going to support that here. But the HTTP verbs and zero vs some + args makes sense - it lets you define custom forms pretty easily. + + Moreover, I'm of the opinion that empty overload really only makes + sense on GET for this case. On a POST, it is just a missing argument + exception and that should be handled by the presenter. But meh, I'll + let the user define that, D only allows one empty arg thing anyway + so the method UDAs are irrelevant. + +/ + if(callFunction) + +/ + + auto format = cgi.request("format", defaultFormat!overload()); + auto wantsFormFormat = format.startsWith("form-"); + + if(wantsFormFormat || (automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET)) { + // Should I still show the form on a json thing? idk... + auto ret = presenter.createAutomaticFormForFunction!((__traits(getOverloads, obj, methodName)[idx]))(&(__traits(getOverloads, obj, methodName)[idx])); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), wantsFormFormat ? format["form_".length .. $] : "html"); + return true; + } + + try { + // a void return (or typeof(null) lol) means you, the user, is doing it yourself. Gives full control. + auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); + } catch(Throwable t) { + // presenter.presentExceptionAsHtml!(__traits(getOverloads, obj, methodName)[idx])(cgi, t, &(__traits(getOverloads, obj, methodName)[idx])); + presenter.presentExceptionalReturn(cgi, t, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); + } + return true; + //}} + + //cgi.header("Accept: POST"); // FIXME list the real thing + //cgi.setResponseStatus("405 Method Not Allowed"); // again, not exactly, but sort of. no overload matched our args, almost certainly due to http verb filtering. + //return true; + } + } + } + case "GET script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(presenter.script(), true); + return true; + case "GET style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(presenter.style(), true); + return true; + default: + return false; + } + + assert(0); + } + return DispatcherDefinition!internalHandler(urlPrefix, false); +} + +string defaultFormat(alias method)() { + bool nonConstConditionForWorkingAroundASpuriousDmdWarning = true; + foreach(attr; __traits(getAttributes, method)) { + static if(is(typeof(attr) == DefaultFormat)) { + if(nonConstConditionForWorkingAroundASpuriousDmdWarning) + return attr.value; + } + } + return "html"; +} + +struct Paginated(T) { + T[] items; + string nextPageUrl; +} + +template urlNamesForMethod(alias method, string default_) { + string[] helper() { + auto verb = Cgi.RequestMethod.GET; + bool foundVerb = false; + bool foundNoun = false; + + string def = default_; + + bool hasAutomaticForm = false; + + foreach(attr; __traits(getAttributes, method)) { + static if(is(typeof(attr) == Cgi.RequestMethod)) { + verb = attr; + if(foundVerb) + assert(0, "Multiple http verbs on one function is not currently supported"); + foundVerb = true; + } + static if(is(typeof(attr) == UrlName)) { + if(foundNoun) + assert(0, "Multiple url names on one function is not currently supported"); + foundNoun = true; + def = attr.name; + } + static if(__traits(isSame, attr, AutomaticForm)) { + hasAutomaticForm = true; + } + } + + if(def is null) + def = "__null"; + + string[] ret; + + static if(is(typeof(method) R == return)) { + static if(is(R : WebObject)) { + def ~= "/"; + foreach(v; __traits(allMembers, Cgi.RequestMethod)) + ret ~= v ~ " " ~ def; + } else { + if(hasAutomaticForm) { + ret ~= "GET " ~ def; + ret ~= "POST " ~ def; + } else { + ret ~= to!string(verb) ~ " " ~ def; + } + } + } else static assert(0); + + return ret; + } + enum urlNamesForMethod = helper(); +} + + + enum AccessCheck { + allowed, + denied, + nonExistant, + } + + enum Operation { + show, + create, + replace, + remove, + update + } + + enum UpdateResult { + accessDenied, + noSuchResource, + success, + failure, + unnecessary + } + + enum ValidationResult { + valid, + invalid + } + + +/++ + The base of all REST objects, to be used with [serveRestObject] and [serveRestCollectionOf]. + + WARNING: this is not stable. ++/ +class RestObject(CRTP) : WebObject { + + import arsd.dom; + import arsd.jsvar; + + /// Prepare the object to be shown. + void show() {} + /// ditto + void show(string urlId) { + load(urlId); + show(); + } + + /// Override this to provide access control to this object. + AccessCheck accessCheck(string urlId, Operation operation) { + return AccessCheck.allowed; + } + + ValidationResult validate() { + // FIXME + return ValidationResult.valid; + } + + string getUrlSlug() { + import std.conv; + static if(is(typeof(CRTP.id))) + return to!string((cast(CRTP) this).id); + else + return null; + } + + // The functions with more arguments are the low-level ones, + // they forward to the ones with fewer arguments by default. + + // POST on a parent collection - this is called from a collection class after the members are updated + /++ + Given a populated object, this creates a new entry. Returns the url identifier + of the new object. + +/ + string create(scope void delegate() applyChanges) { + applyChanges(); + save(); + return getUrlSlug(); + } + + void replace() { + save(); + } + void replace(string urlId, scope void delegate() applyChanges) { + load(urlId); + applyChanges(); + replace(); + } + + void update(string[] fieldList) { + save(); + } + void update(string urlId, scope void delegate() applyChanges, string[] fieldList) { + load(urlId); + applyChanges(); + update(fieldList); + } + + void remove() {} + + void remove(string urlId) { + load(urlId); + remove(); + } + + abstract void load(string urlId); + abstract void save(); + + Element toHtml(Presenter)(Presenter presenter) { + import arsd.dom; + import std.conv; + auto obj = cast(CRTP) this; + auto div = Element.make("div"); + div.addClass("Dclass_" ~ CRTP.stringof); + div.dataset.url = getUrlSlug(); + bool first = true; + foreach(idx, memberName; __traits(derivedMembers, CRTP)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + if(!first) div.addChild("br"); else first = false; + div.appendChild(presenter.formatReturnValueAsHtml(__traits(getMember, obj, memberName))); + } + return div; + } + + var toJson() { + import arsd.jsvar; + var v = var.emptyObject(); + auto obj = cast(CRTP) this; + foreach(idx, memberName; __traits(derivedMembers, CRTP)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + v[memberName] = __traits(getMember, obj, memberName); + } + return v; + } + + /+ + auto structOf(this This) { + + } + +/ +} + +// FIXME XSRF token, prolly can just put in a cookie and then it needs to be copied to header or form hidden value +// https://use-the-index-luke.com/sql/partial-results/fetch-next-page + +/++ + Base class for REST collections. ++/ +class CollectionOf(Obj) : RestObject!(CollectionOf) { + /// You might subclass this and use the cgi object's query params + /// to implement a search filter, for example. + /// + /// FIXME: design a way to auto-generate that form + /// (other than using the WebObject thing above lol + // it'll prolly just be some searchParams UDA or maybe an enum. + // + // pagination too perhaps. + // + // and sorting too + IndexResult index() { return IndexResult.init; } + + string[] sortableFields() { return null; } + string[] searchableFields() { return null; } + + struct IndexResult { + Obj[] results; + + string[] sortableFields; + + string previousPageIdentifier; + string nextPageIdentifier; + string firstPageIdentifier; + string lastPageIdentifier; + + int numberOfPages; + } + + override string create(scope void delegate() applyChanges) { assert(0); } + override void load(string urlId) { assert(0); } + override void save() { assert(0); } + override void show() { + index(); + } + override void show(string urlId) { + show(); + } + + /// Proxy POST requests (create calls) to the child collection + alias PostProxy = Obj; +} + +/++ + Serves a REST object, similar to a Ruby on Rails resource. + + You put data members in your class. cgi.d will automatically make something out of those. + + It will call your constructor with the ID from the URL. This may be null. + It will then populate the data members from the request. + It will then call a method, if present, telling what happened. You don't need to write these! + It finally returns a reply. + + Your methods are passed a list of fields it actually set. + + The URL mapping - despite my general skepticism of the wisdom - matches up with what most REST + APIs I have used seem to follow. (I REALLY want to put trailing slashes on it though. Works better + with relative linking. But meh.) + + GET /items -> index. all values not set. + GET /items/id -> get. only ID will be set, other params ignored. + POST /items -> create. values set as given + PUT /items/id -> replace. values set as given + or POST /items/id with cgi.post["_method"] (thus urlencoded or multipart content-type) set to "PUT" to work around browser/html limitation + a GET with cgi.get["_method"] (in the url) set to "PUT" will render a form. + PATCH /items/id -> update. values set as given, list of changed fields passed + or POST /items/id with cgi.post["_method"] == "PATCH" + DELETE /items/id -> destroy. only ID guaranteed to be set + or POST /items/id with cgi.post["_method"] == "DELETE" + + Following the stupid convention, there will never be a trailing slash here, and if it is there, it will + redirect you away from it. + + API clients should set the `Accept` HTTP header to application/json or the cgi.get["_format"] = "json" var. + + I will also let you change the default, if you must. + + // One add-on is validation. You can issue a HTTP GET to a resource with _method = VALIDATE to check potential changes. + + You can define sub-resources on your object inside the object. These sub-resources are also REST objects + that follow the same thing. They may be individual resources or collections themselves. + + Your class is expected to have at least the following methods: + + FIXME: i kinda wanna add a routes object to the initialize call + + create + Create returns the new address on success, some code on failure. + show + index + update + remove + + You will want to be able to customize the HTTP, HTML, and JSON returns but generally shouldn't have to - the defaults + should usually work. The returned JSON will include a field "href" on all returned objects along with "id". Or omething like that. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + NOT IMPLEMENTED + + + Really, a collection is a resource with a bunch of subresources. + + GET /items + index because it is GET on the top resource + + GET /items/foo + item but different than items? + + class Items { + + } + + ... but meh, a collection can be automated. not worth making it + a separate thing, let's look at a real example. Users has many + items and a virtual one, /users/current. + + the individual users have properties and two sub-resources: + session, which is just one, and comments, a collection. + + class User : RestObject!() { // no parent + int id; + string name; + + // the default implementations of the urlId ones is to call load(that_id) then call the arg-less one. + // but you can override them to do it differently. + + // any member which is of type RestObject can be linked automatically via href btw. + + void show() {} + void show(string urlId) {} // automated! GET of this specific thing + void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + void replace(string urlId) {} // this is the PUT; really, it just updates all fields. + void update(string urlId, string[] fieldList) {} // PATCH, it updates some fields. + void remove(string urlId) {} // DELETE + + void load(string urlId) {} // the default implementation of show() populates the id, then + + this() {} + + mixin Subresource!Session; + mixin Subresource!Comment; + } + + class Session : RestObject!() { + // the parent object may not be fully constructed/loaded + this(User parent) {} + + } + + class Comment : CollectionOf!Comment { + this(User parent) {} + } + + class Users : CollectionOf!User { + // but you don't strictly need ANYTHING on a collection; it will just... collect. Implement the subobjects. + void index() {} // GET on this specific thing; just like show really, just different name for the different semantics. + User create() {} // You MAY implement this, but the default is to create a new object, populate it from args, and then call create() on the child + } + ++/ +auto serveRestObject(T)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$ - 1] != '/', "Do NOT use a trailing slash on REST objects."); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, immutable void* details) { + string url = cgi.pathInfo[urlPrefix.length .. $]; + + if(url.length && url[$ - 1] == '/') { + // remove the final slash... + cgi.setResponseLocation(cgi.scriptName ~ cgi.pathInfo[0 .. $ - 1]); + return true; + } + + return restObjectServeHandler!T(cgi, presenter, url); + } + return DispatcherDefinition!internalHandler(urlPrefix, false); +} + +/+ +/// Convenience method for serving a collection. It will be named the same +/// as type T, just with an s at the end. If you need any further, just +/// write the class yourself. +auto serveRestCollectionOf(T)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + mixin(`static class `~T.stringof~`s : CollectionOf!(T) {}`); + return serveRestObject!(mixin(T.stringof ~ "s"))(urlPrefix); +} ++/ + +bool restObjectServeHandler(T, Presenter)(Cgi cgi, Presenter presenter, string url) { + string urlId = null; + if(url.length && url[0] == '/') { + // asking for a subobject + urlId = url[1 .. $]; + foreach(idx, ch; urlId) { + if(ch == '/') { + urlId = urlId[0 .. idx]; + break; + } + } + } + + // FIXME handle other subresources + + static if(is(T : CollectionOf!(C), C)) { + if(urlId !is null) { + return restObjectServeHandler!(C, Presenter)(cgi, presenter, url); // FIXME? urlId); + } + } + + // FIXME: support precondition failed, if-modified-since, expectation failed, etc. + + auto obj = new T(); + obj.initialize(cgi); + // FIXME: populate reflection info delegates + + + // FIXME: I am not happy with this. + switch(urlId) { + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(presenter.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(presenter.style(), true); + return true; + default: + // intentionally blank + } + + + + + static void applyChangesTemplate(Obj)(Cgi cgi, Obj obj) { + foreach(idx, memberName; __traits(derivedMembers, Obj)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + __traits(getMember, obj, memberName) = cgi.request(memberName, __traits(getMember, obj, memberName)); + } + } + void applyChanges() { + applyChangesTemplate(cgi, obj); + } + + string[] modifiedList; + + void writeObject(bool addFormLinks) { + if(cgi.request("format") == "json") { + cgi.setResponseContentType("application/json"); + cgi.write(obj.toJson().toString, true); + } else { + auto container = presenter.htmlContainer(); + if(addFormLinks) { + static if(is(T : CollectionOf!(C), C)) + container.appendHtml(` + + + + `); + else + container.appendHtml(` + Back +
    + + +
    + `); + } + container.appendChild(obj.toHtml(presenter)); + cgi.write(container.parentDocument.toString, true); + } + } + + // FIXME: I think I need a set type in here.... + // it will be nice to pass sets of members. + + try + switch(cgi.requestMethod) { + case Cgi.RequestMethod.GET: + // I could prolly use template this parameters in the implementation above for some reflection stuff. + // sure, it doesn't automatically work in subclasses... but I instantiate here anyway... + + // automatic forms here for usable basic auto site from browser. + // even if the format is json, it could actually send out the links and formats, but really there i'ma be meh. + switch(cgi.request("_method", "GET")) { + case "GET": + static if(is(T : CollectionOf!(C), C)) { + auto results = obj.index(); + if(cgi.request("format", "html") == "html") { + auto container = presenter.htmlContainer(); + auto html = presenter.formatReturnValueAsHtml(results.results); + container.appendHtml(` +
    + +
    + `); + + container.appendChild(html); + cgi.write(container.parentDocument.toString, true); + } else { + cgi.setResponseContentType("application/json"); + import arsd.jsvar; + var json = var.emptyArray; + foreach(r; results.results) { + var o = var.emptyObject; + foreach(idx, memberName; __traits(derivedMembers, typeof(r))) + static if(__traits(compiles, __traits(getMember, r, memberName).offsetof)) { + o[memberName] = __traits(getMember, r, memberName); + } + + json ~= o; + } + cgi.write(json.toJson(), true); + } + } else { + obj.show(urlId); + writeObject(true); + } + break; + case "PATCH": + obj.load(urlId); + goto case; + case "PUT": + case "POST": + // an editing form for the object + auto container = presenter.htmlContainer(); + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto form = (cgi.request("_method") == "POST") ? presenter.createAutomaticFormForObject(new obj.PostProxy()) : presenter.createAutomaticFormForObject(obj); + } else { + auto form = presenter.createAutomaticFormForObject(obj); + } + form.attrs.method = "POST"; + form.setValue("_method", cgi.request("_method", "GET")); + container.appendChild(form); + cgi.write(container.parentDocument.toString(), true); + break; + case "DELETE": + // FIXME: a delete form for the object (can be phrased "are you sure?") + auto container = presenter.htmlContainer(); + container.appendHtml(` +
    + Are you sure you want to delete this item? + + +
    + + `); + cgi.write(container.parentDocument.toString(), true); + break; + default: + cgi.write("bad method\n", true); + } + break; + case Cgi.RequestMethod.POST: + // this is to allow compatibility with HTML forms + switch(cgi.request("_method", "POST")) { + case "PUT": + goto PUT; + case "PATCH": + goto PATCH; + case "DELETE": + goto DELETE; + case "POST": + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto p = new obj.PostProxy(); + void specialApplyChanges() { + applyChangesTemplate(cgi, p); + } + string n = p.create(&specialApplyChanges); + } else { + string n = obj.create(&applyChanges); + } + + auto newUrl = cgi.scriptName ~ cgi.pathInfo ~ "/" ~ n; + cgi.setResponseLocation(newUrl); + cgi.setResponseStatus("201 Created"); + cgi.write(`The object has been created.`); + break; + default: + cgi.write("bad method\n", true); + } + // FIXME this should be valid on the collection, but not the child.... + // 303 See Other + break; + case Cgi.RequestMethod.PUT: + PUT: + obj.replace(urlId, &applyChanges); + writeObject(false); + break; + case Cgi.RequestMethod.PATCH: + PATCH: + obj.update(urlId, &applyChanges, modifiedList); + writeObject(false); + break; + case Cgi.RequestMethod.DELETE: + DELETE: + obj.remove(urlId); + cgi.setResponseStatus("204 No Content"); + break; + default: + // FIXME: OPTIONS, HEAD + } + catch(Throwable t) { + presenter.presentExceptionAsHtml(cgi, t); + } + + return true; +} + +/+ +struct SetOfFields(T) { + private void[0][string] storage; + void set(string what) { + //storage[what] = + } + void unset(string what) {} + void setAll() {} + void unsetAll() {} + bool isPresent(string what) { return false; } +} ++/ + +/+ +enum readonly; +enum hideonindex; ++/ + +/++ + Returns true if I recommend gzipping content of this type. You might + want to call it from your Presenter classes before calling cgi.write. + + --- + cgi.setResponseContentType(yourContentType); + cgi.gzipResponse = gzipRecommendedForContentType(yourContentType); + cgi.write(yourData, true); + --- + + This is used internally by [serveStaticFile], [serveStaticFileDirectory], [serveStaticData], and maybe others I forgot to update this doc about. + + + The implementation considers text content to be recommended to gzip. This may change, but it seems reasonable enough for now. + + History: + Added January 28, 2023 (dub v11.0) ++/ +bool gzipRecommendedForContentType(string contentType) { + if(contentType.startsWith("text/")) + return true; + if(contentType.startsWith("application/javascript")) + return true; + + return false; +} + +/++ + Serves a static file. To be used with [dispatcher]. + + See_Also: [serveApi], [serveRestObject], [dispatcher], [serveRedirect] ++/ +auto serveStaticFile(string urlPrefix, string filename = null, string contentType = null) { +// https://baus.net/on-tcp_cork/ +// man 2 sendfile + assert(urlPrefix[0] == '/'); + if(filename is null) + filename = decodeUriComponent(urlPrefix[1 .. $]); // FIXME is this actually correct? + if(contentType is null) { + contentType = contentTypeFromFileExtension(filename); + } + + static struct DispatcherDetails { + string filename; + string contentType; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + if(details.contentType.indexOf("image/") == 0 || details.contentType.indexOf("audio/") == 0) + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); + cgi.write(std.file.read(details.filename), true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(filename, contentType)); +} + +/++ + Serves static data. To be used with [dispatcher]. + + History: + Added October 31, 2021 ++/ +auto serveStaticData(string urlPrefix, immutable(void)[] data, string contentType = null) { + assert(urlPrefix[0] == '/'); + if(contentType is null) { + contentType = contentTypeFromFileExtension(urlPrefix); + } + + static struct DispatcherDetails { + immutable(void)[] data; + string contentType; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + cgi.write(details.data, true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(data, contentType)); +} + +string contentTypeFromFileExtension(string filename) { + if(filename.endsWith(".png")) + return "image/png"; + if(filename.endsWith(".apng")) + return "image/apng"; + if(filename.endsWith(".svg")) + return "image/svg+xml"; + if(filename.endsWith(".jpg")) + return "image/jpeg"; + if(filename.endsWith(".html")) + return "text/html"; + if(filename.endsWith(".css")) + return "text/css"; + if(filename.endsWith(".js")) + return "application/javascript"; + if(filename.endsWith(".wasm")) + return "application/wasm"; + if(filename.endsWith(".mp3")) + return "audio/mpeg"; + if(filename.endsWith(".pdf")) + return "application/pdf"; + return null; +} + +/// This serves a directory full of static files, figuring out the content-types from file extensions. +/// It does not let you to descend into subdirectories (or ascend out of it, of course) +auto serveStaticFileDirectory(string urlPrefix, string directory = null, bool recursive = false) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] == '/'); + + static struct DispatcherDetails { + string directory; + bool recursive; + } + + if(directory is null) + directory = urlPrefix[1 .. $]; + + if(directory.length == 0) + directory = "./"; + + assert(directory[$-1] == '/'); + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + auto file = decodeUriComponent(cgi.pathInfo[urlPrefix.length .. $]); // FIXME: is this actually correct + + if(details.recursive) { + // never allow a backslash since it isn't in a typical url anyway and makes the following checks easier + if(file.indexOf("\\") != -1) + return false; + + import std.path; + + file = std.path.buildNormalizedPath(file); + enum upOneDir = ".." ~ std.path.dirSeparator; + + // also no point doing any kind of up directory things since that makes it more likely to break out of the parent + if(file == ".." || file.startsWith(upOneDir)) + return false; + if(std.path.isAbsolute(file)) + return false; + + // FIXME: if it has slashes and stuff, should we redirect to the canonical resource? or what? + + // once it passes these filters it is probably ok. + } else { + if(file.indexOf("/") != -1 || file.indexOf("\\") != -1) + return false; + } + + if(file.length == 0) + return false; + + auto contentType = contentTypeFromFileExtension(file); + + auto fn = details.directory ~ file; + if(std.file.exists(fn)) { + //if(contentType.indexOf("image/") == 0) + //cgi.setCache(true); + //else if(contentType.indexOf("audio/") == 0) + cgi.setCache(true); + cgi.setResponseContentType(contentType); + cgi.gzipResponse = gzipRecommendedForContentType(contentType); + cgi.write(std.file.read(fn), true); + return true; + } else { + return false; + } + } + + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, false, DispatcherDetails(directory, recursive)); +} + +/++ + Redirects one url to another + + See_Also: [dispatcher], [serveStaticFile] ++/ +auto serveRedirect(string urlPrefix, string redirectTo, int code = 303) { + assert(urlPrefix[0] == '/'); + static struct DispatcherDetails { + string redirectTo; + string code; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setResponseLocation(details.redirectTo, true, details.code); + return true; + } + + + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(redirectTo, getHttpCodeText(code))); +} + +/// Used exclusively with `dispatchTo` +struct DispatcherData(Presenter) { + Cgi cgi; /// You can use this cgi object. + Presenter presenter; /// This is the presenter from top level, and will be forwarded to the sub-dispatcher. + size_t pathInfoStart; /// This is forwarded to the sub-dispatcher. It may be marked private later, or at least read-only. +} + +/++ + Dispatches the URL to a specific function. ++/ +auto handleWith(alias handler)(string urlPrefix) { + // cuz I'm too lazy to do it better right now + static class Hack : WebObject { + static import std.traits; + @UrlName("") + auto handle(std.traits.Parameters!handler args) { + return handler(args); + } + } + + return urlPrefix.serveApiInternal!Hack; +} + +/++ + Dispatches the URL (and anything under it) to another dispatcher function. The function should look something like this: + + --- + bool other(DD)(DD dd) { + return dd.dispatcher!( + "/whatever".serveRedirect("/success"), + "/api/".serveApi!MyClass + ); + } + --- + + The `DD` in there will be an instance of [DispatcherData] which you can inspect, or forward to another dispatcher + here. It is a template to account for any Presenter type, so you can do compile-time analysis in your presenters. + Or, of course, you could just use the exact type in your own code. + + You return true if you handle the given url, or false if not. Just returning the result of [dispatcher] will do a + good job. + + ++/ +auto dispatchTo(alias handler)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] != '/'); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, const void* details) { + return handler(DispatcherData!Presenter(cgi, presenter, urlPrefix.length)); + } + + return DispatcherDefinition!(internalHandler)(urlPrefix, false); +} + +/++ + See [serveStaticFile] if you want to serve a file off disk. + + History: + Added January 28, 2023 (dub v11.0) ++/ +auto serveStaticData(string urlPrefix, immutable(ubyte)[] data, string contentType, string filenameToSuggestAsDownload = null) { + assert(urlPrefix[0] == '/'); + + static struct DispatcherDetails { + immutable(ubyte)[] data; + string contentType; + string filenameToSuggestAsDownload; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + if(details.filenameToSuggestAsDownload.length) + cgi.header("Content-Disposition: attachment; filename=\""~details.filenameToSuggestAsDownload~"\""); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); + cgi.write(details.data, true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(data, contentType, filenameToSuggestAsDownload)); +} + +/++ + Placeholder for use with [dispatchSubsection]'s `NewPresenter` argument to indicate you want to keep the parent's presenter. + + History: + Added January 28, 2023 (dub v11.0) ++/ +alias KeepExistingPresenter = typeof(null); + +/++ + For use with [dispatchSubsection]. Calls your filter with the request and if your filter returns false, + this issues the given errorCode and stops processing. + + --- + bool hasAdminPermissions(Cgi cgi) { + return true; + } + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + passFilterOrIssueError!(hasAdminPermissions, 403), + KeepExistingPresenter, + "/".serveApi!AdminFunctions + ) + ); + --- + + History: + Added January 28, 2023 (dub v11.0) ++/ +template passFilterOrIssueError(alias filter, int errorCode) { + bool passFilterOrIssueError(DispatcherDetails)(DispatcherDetails dd) { + if(filter(dd.cgi)) + return true; + dd.presenter.renderBasicError(dd.cgi, errorCode); + return false; + } +} + +/++ + Allows for a subsection of your dispatched urls to be passed through other a pre-request filter, optionally pick up an new presenter class, + and then be dispatched to their own handlers. + + --- + /+ + // a long-form filter function + bool permissionCheck(DispatcherData)(DispatcherData dd) { + // you are permitted to call mutable methods on the Cgi object + // Note if you use a Cgi subclass, you can try dynamic casting it back to your custom type to attach per-request data + // though much of the request is immutable so there's only so much you're allowed to do to modify it. + + if(checkPermissionOnRequest(dd.cgi)) { + return true; // OK, allow processing to continue + } else { + dd.presenter.renderBasicError(dd.cgi, 403); // reply forbidden to the requester + return false; // and stop further processing into this subsection + } + } + +/ + + // but you can also do short-form filters: + + bool permissionCheck(Cgi cgi) { + return ("ok" in cgi.get) !is null; + } + + // handler for the subsection + class AdminClass : WebObject { + int foo() { return 5; } + } + + // handler for the main site + class TheMainSite : WebObject {} + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + // converts our short-form filter into a long-form filter + passFilterOrIssueError!(permissionCheck, 403), + // can use a new presenter if wanted for the subsection + KeepExistingPresenter, + // and then provide child route dispatchers + "/".serveApi!AdminClass + ), + // and back to the top level + "/".serveApi!TheMainSite + ); + --- + + Note you can encapsulate sections in files like this: + + --- + auto adminDispatcher(string urlPrefix) { + return urlPrefix.dispatchSubsection!( + .... + ); + } + + mixin DispatcherMain!( + "/admin".adminDispatcher, + // and so on + ) + --- + + If you want no filter, you can pass `(cgi) => true` as the filter to approve all requests. + + If you want to keep the same presenter as the parent, use [KeepExistingPresenter] as the presenter argument. + + + History: + Added January 28, 2023 (dub v11.0) ++/ +auto dispatchSubsection(alias PreRequestFilter, NewPresenter, definitions...)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] != '/'); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, const void* details) { + static if(!is(PreRequestFilter == typeof(null))) { + if(!PreRequestFilter(DispatcherData!Presenter(cgi, presenter, urlPrefix.length))) + return true; // we handled it by rejecting it + } + + static if(is(NewPresenter == Presenter) || is(NewPresenter == typeof(null))) { + return dispatcher!definitions(DispatcherData!Presenter(cgi, presenter, urlPrefix.length)); + } else { + auto newPresenter = new NewPresenter(); + return dispatcher!(definitions(DispatcherData!NewPresenter(cgi, presenter, urlPrefix.length))); + } + } + + return DispatcherDefinition!(internalHandler)(urlPrefix, false); +} + +/++ + A URL dispatcher. + + --- + if(cgi.dispatcher!( + "/api/".serveApi!MyApiClass, + "/objects/lol".serveRestObject!MyRestObject, + "/file.js".serveStaticFile, + "/admin/".dispatchTo!adminHandler + )) return; + --- + + + You define a series of url prefixes followed by handlers. + + You may want to do different pre- and post- processing there, for example, + an authorization check and different page layout. You can use different + presenters and different function chains. See [dispatchSubsection] for details. + + [dispatchTo] will send the request to another function for handling. ++/ +template dispatcher(definitions...) { + bool dispatcher(Presenter)(Cgi cgi, Presenter presenterArg = null) { + static if(is(Presenter == typeof(null))) { + static class GenericWebPresenter : WebPresenter!(GenericWebPresenter) {} + auto presenter = new GenericWebPresenter(); + } else + alias presenter = presenterArg; + + return dispatcher(DispatcherData!(typeof(presenter))(cgi, presenter, 0)); + } + + bool dispatcher(DispatcherData)(DispatcherData dispatcherData) if(!is(DispatcherData : Cgi)) { + // I can prolly make this more efficient later but meh. + foreach(definition; definitions) { + if(definition.rejectFurther) { + if(dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart .. $] == definition.urlPrefix) { + auto ret = definition.handler( + dispatcherData.cgi.pathInfo[0 .. dispatcherData.pathInfoStart + definition.urlPrefix.length], + dispatcherData.cgi, dispatcherData.presenter, definition.details); + if(ret) + return true; + } + } else if( + dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart .. $].startsWith(definition.urlPrefix) && + // cgi.d dispatcher urls must be complete or have a /; + // "foo" -> thing should NOT match "foobar", just "foo" or "foo/thing" + (definition.urlPrefix[$-1] == '/' || (dispatcherData.pathInfoStart + definition.urlPrefix.length) == dispatcherData.cgi.pathInfo.length + || dispatcherData.cgi.pathInfo[dispatcherData.pathInfoStart + definition.urlPrefix.length] == '/') + ) { + auto ret = definition.handler( + dispatcherData.cgi.pathInfo[0 .. dispatcherData.pathInfoStart + definition.urlPrefix.length], + dispatcherData.cgi, dispatcherData.presenter, definition.details); + if(ret) + return true; + } + } + return false; + } +} + +}); + +private struct StackBuffer { + char[1024] initial = void; + char[] buffer; + size_t position; + + this(int a) { + buffer = initial[]; + position = 0; + } + + void add(in char[] what) { + if(position + what.length > buffer.length) + buffer.length = position + what.length + 1024; // reallocate with GC to handle special cases + buffer[position .. position + what.length] = what[]; + position += what.length; + } + + void add(in char[] w1, in char[] w2, in char[] w3 = null) { + add(w1); + add(w2); + add(w3); + } + + void add(long v) { + char[16] buffer = void; + auto pos = buffer.length; + bool negative; + if(v < 0) { + negative = true; + v = -v; + } + do { + buffer[--pos] = cast(char) (v % 10 + '0'); + v /= 10; + } while(v); + + if(negative) + buffer[--pos] = '-'; + + auto res = buffer[pos .. $]; + + add(res[]); + } + + char[] get() @nogc { + return buffer[0 .. position]; + } +} + +// duplicated in http2.d +private static string getHttpCodeText(int code) pure nothrow @nogc { + switch(code) { + case 200: return "200 OK"; + case 201: return "201 Created"; + case 202: return "202 Accepted"; + case 203: return "203 Non-Authoritative Information"; + case 204: return "204 No Content"; + case 205: return "205 Reset Content"; + case 206: return "206 Partial Content"; + // + case 300: return "300 Multiple Choices"; + case 301: return "301 Moved Permanently"; + case 302: return "302 Found"; + case 303: return "303 See Other"; + case 304: return "304 Not Modified"; + case 305: return "305 Use Proxy"; + case 307: return "307 Temporary Redirect"; + case 308: return "308 Permanent Redirect"; + + // + case 400: return "400 Bad Request"; + case 401: return "401 Unauthorized"; + case 402: return "402 Payment Required"; + case 403: return "403 Forbidden"; + case 404: return "404 Not Found"; + case 405: return "405 Method Not Allowed"; + case 406: return "406 Not Acceptable"; + case 407: return "407 Proxy Authentication Required"; + case 408: return "408 Request Timeout"; + case 409: return "409 Conflict"; + case 410: return "410 Gone"; + case 411: return "411 Length Required"; + case 412: return "412 Precondition Failed"; + case 413: return "413 Payload Too Large"; + case 414: return "414 URI Too Long"; + case 415: return "415 Unsupported Media Type"; + case 416: return "416 Range Not Satisfiable"; + case 417: return "417 Expectation Failed"; + case 418: return "418 I'm a teapot"; + case 421: return "421 Misdirected Request"; + case 422: return "422 Unprocessable Entity (WebDAV)"; + case 423: return "423 Locked (WebDAV)"; + case 424: return "424 Failed Dependency (WebDAV)"; + case 425: return "425 Too Early"; + case 426: return "426 Upgrade Required"; + case 428: return "428 Precondition Required"; + case 431: return "431 Request Header Fields Too Large"; + case 451: return "451 Unavailable For Legal Reasons"; + + case 500: return "500 Internal Server Error"; + case 501: return "501 Not Implemented"; + case 502: return "502 Bad Gateway"; + case 503: return "503 Service Unavailable"; + case 504: return "504 Gateway Timeout"; + case 505: return "505 HTTP Version Not Supported"; + case 506: return "506 Variant Also Negotiates"; + case 507: return "507 Insufficient Storage (WebDAV)"; + case 508: return "508 Loop Detected (WebDAV)"; + case 510: return "510 Not Extended"; + case 511: return "511 Network Authentication Required"; + // + default: assert(0, "Unsupported http code"); + } +} + + +/+ +/++ + This is the beginnings of my web.d 2.0 - it dispatches web requests to a class object. + + It relies on jsvar.d and dom.d. + + + You can get javascript out of it to call. The generated functions need to look + like + + function name(a,b,c,d,e) { + return _call("name", {"realName":a,"sds":b}); + } + + And _call returns an object you can call or set up or whatever. ++/ +bool apiDispatcher()(Cgi cgi) { + import arsd.jsvar; + import arsd.dom; +} ++/ +version(linux) +private extern(C) int eventfd (uint initval, int flags) nothrow @trusted @nogc; +/* +Copyright: Adam D. Ruppe, 2008 - 2023 +License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0]. +Authors: Adam D. Ruppe + + Copyright Adam D. Ruppe 2008 - 2023. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ diff --git a/arsd/core.d b/arsd/core.d new file mode 100644 index 0000000..2e26b58 --- /dev/null +++ b/arsd/core.d @@ -0,0 +1,8581 @@ +/++ + $(PITFALL + Please note: the api and behavior of this module is not externally stable at this time. See the documentation on specific functions for details. + ) + + Shared core functionality including exception helpers, library loader, event loop, and possibly more. Maybe command line processor and uda helper and some basic shared annotation types. + + I'll probably move the url, websocket, and ssl stuff in here too as they are often shared. Maybe a small internationalization helper type (a hook for external implementation) and COM helpers too. I might move the process helpers out to their own module - even things in here are not considered stable to library users at this time! + + If you use this directly outside the arsd library despite its current instability caveats, you might consider using `static import` since names in here are likely to clash with Phobos if you use them together. `static import` will let you easily disambiguate and avoid name conflict errors if I add more here. Some names even clash deliberately to remind me to avoid some antipatterns inside the arsd modules! + + ## Contributor notes + + arsd.core should be focused on things that enable interoperability primarily and secondarily increased code quality between other, otherwise independent arsd modules. As a foundational library, it is not permitted to import anything outside the druntime `core` namespace, except in templates and examples not normally compiled in. This keeps it independent and avoids transitive dependency spillover to end users while also keeping compile speeds fast. To help keep builds snappy, also avoid significant use of ctfe inside this module. + + On my linux computer, `dmd -unittest -main core.d` takes about a quarter second to run. We do not want this to grow. + + `@safe` compatibility is ok when it isn't too big of a hassle. `@nogc` is a non-goal. I might accept it on some of the trivial functions but if it means changing the logic in any way to support, you will need a compelling argument to justify it. The arsd libs are supposed to be reliable and easy to use. That said, of course, don't be unnecessarily wasteful - if you can easily provide a reliable and easy to use way to let advanced users do their thing without hurting the other cases, let's discuss it. + + If functionality is not needed by multiple existing arsd modules, consider adding a new module instead of adding it to the core. + + Unittests should generally be hidden behind a special version guard so they don't interfere with end user tests. + + History: + Added March 2023 (dub v11.0). Several functions were migrated in here at that time, noted individually. Members without a note were added with the module. ++/ +module arsd.core; + + +static if(__traits(compiles, () { import core.interpolation; })) { + import core.interpolation; + + alias InterpolationHeader = core.interpolation.InterpolationHeader; + alias InterpolationFooter = core.interpolation.InterpolationFooter; + alias InterpolatedLiteral = core.interpolation.InterpolatedLiteral; + alias InterpolatedExpression = core.interpolation.InterpolatedExpression; +} else { + // polyfill for old versions + struct InterpolationHeader {} + struct InterpolationFooter {} + struct InterpolatedLiteral(string literal) {} + struct InterpolatedExpression(string code) {} +} + +version(use_arsd_core) + enum use_arsd_core = true; +else + enum use_arsd_core = false; + +import core.attribute; +static if(__traits(hasMember, core.attribute, "implicit")) + alias implicit = core.attribute.implicit; +else + enum implicit; + + +// FIXME: add callbacks on file open for tracing dependencies dynamically + +// see for useful info: https://devblogs.microsoft.com/dotnet/how-async-await-really-works/ + +// see: https://wiki.openssl.org/index.php/Simple_TLS_Server + +// see: When you only want to track changes on a file or directory, be sure to open it using the O_EVTONLY flag. + +///ArsdUseCustomRuntime is used since other derived work from WebAssembly may be used and thus specified in the CLI +version(WebAssembly) version = ArsdUseCustomRuntime; + +// note that kqueue might run an i/o loop on mac, ios, etc. but then NSApp.run on the io thread +// but on bsd, you want the kqueue loop in i/o too.... + +version(iOS) +{ + version = EmptyEventLoop; + version = EmptyCoreEvent; +} +version(ArsdUseCustomRuntime) +{ + version = EmptyEventLoop; + version = UseStdioWriteln; +} +else +{ + version = HasFile; + version = HasSocket; + version = HasThread; + version = HasErrno; + + version(Windows) + version = HasTimer; + version(linux) + version = HasTimer; + version(OSXCocoa) + version = HasTimer; +} + +version(HasThread) +{ + import core.thread; + import core.volatile; + import core.atomic; + import core.time; +} +else +{ + // polyfill for missing core.time + struct Duration { + static Duration max() { return Duration(); } + } +} + +version(OSX) { + version(ArsdNoCocoa) + enum bool UseCocoa = false; + else + enum bool UseCocoa = true; +} + +version(HasErrno) +import core.stdc.errno; + +import core.attribute; +static if(!__traits(hasMember, core.attribute, "mustuse")) + enum mustuse; + +// FIXME: add an arena allocator? can do task local destruction maybe. + +// the three implementations are windows, epoll, and kqueue +version(Windows) { + version=Arsd_core_windows; + + // import core.sys.windows.windows; + import core.sys.windows.winbase; + import core.sys.windows.windef; + import core.sys.windows.winnls; + import core.sys.windows.winuser; + import core.sys.windows.winsock2; + + pragma(lib, "user32"); + pragma(lib, "ws2_32"); +} else version(linux) { + version=Arsd_core_epoll; + + static if(__VERSION__ >= 2098) { + version=Arsd_core_has_cloexec; + } +} else version(FreeBSD) { + version=Arsd_core_kqueue; + + import core.sys.freebsd.sys.event; + + // the version in druntime doesn't have the default arg making it a pain to use when the freebsd + // version adds a new field + extern(D) void EV_SET(kevent_t* kevp, typeof(kevent_t.tupleof) args = kevent_t.tupleof.init) + { + *kevp = kevent_t(args); + } +} else version(DragonFlyBSD) { + // NOT ACTUALLY TESTED + version=Arsd_core_kqueue; + + import core.sys.dragonflybsd.sys.event; +} else version(NetBSD) { + // NOT ACTUALLY TESTED + version=Arsd_core_kqueue; + + import core.sys.netbsd.sys.event; +} else version(OpenBSD) { + version=Arsd_core_kqueue; + + // THIS FILE DOESN'T ACTUALLY EXIST, WE NEED TO MAKE IT + import core.sys.openbsd.sys.event; +} else version(OSX) { + version=Arsd_core_kqueue; + + import core.sys.darwin.sys.event; + + version(DigitalMars) { + version=OSXCocoa; + } +} + +version(OSXCocoa) + enum CocoaAvailable = true; +else + enum CocoaAvailable = false; + +version(Posix) { + import core.sys.posix.signal; + import core.sys.posix.unistd; + + import core.sys.posix.sys.un; + import core.sys.posix.sys.socket; + import core.sys.posix.netinet.in_; +} + +// FIXME: the exceptions should actually give some explanatory text too (at least sometimes) + +/+ + ========================= + GENERAL UTILITY FUNCTIONS + ========================= ++/ + +/++ + Casts value `v` to type `T`. + + $(TIP + This is a helper function for readability purposes. + The idea is to make type-casting as accessible as `to()` from `std.conv`. + ) + + --- + int i = cast(int)(foo * bar); + int i = castTo!int(foo * bar); + + int j = cast(int) round(floatValue); + int j = round(floatValue).castTo!int; + + int k = cast(int) floatValue + foobar; + int k = floatValue.castTo!int + foobar; + + auto m = Point( + cast(int) calc(a.x, b.x), + cast(int) calc(a.y, b.y), + ); + auto m = Point( + calc(a.x, b.x).castTo!int, + calc(a.y, b.y).castTo!int, + ); + --- + + History: + Added on April 24, 2024. + Renamed from `typeCast` to `castTo` on May 24, 2024. + +/ +auto ref T castTo(T, S)(auto ref S v) { + return cast(T) v; +} + +/// +alias typeCast = castTo; + +// enum stringz : const(char)* { init = null } + +/++ + A wrapper around a `const(char)*` to indicate that it is a zero-terminated C string. ++/ +struct stringz { + private const(char)* raw; + + /++ + Wraps the given pointer in the struct. Note that it retains a copy of the pointer. + +/ + this(const(char)* raw) { + this.raw = raw; + } + + /++ + Returns the original raw pointer back out. + +/ + const(char)* ptr() const { + return raw; + } + + /++ + Borrows a slice of the pointer up to (but not including) the zero terminator. + +/ + const(char)[] borrow() const @system { + if(raw is null) + return null; + + const(char)* p = raw; + int length; + while(*p++) length++; + + return raw[0 .. length]; + } +} + +/+ + DateTime + year: 16 bits (-32k to +32k) + month: 4 bits + day: 5 bits + + hour: 5 bits + minute: 6 bits + second: 6 bits + + total: 25 bits + 17 bits = 42 bits + + fractional seconds: 10 bits + + accuracy flags: date_valid | time_valid = 2 bits + + 54 bits used, 8 bits remain. reserve 1 for signed. + + would need 11 bits for minute-precise dt offset but meh. ++/ + +/++ + A packed date/time/datetime representation added for use with LimitedVariant. + + You should probably not use this much directly, it is mostly an internal storage representation. ++/ +struct PackedDateTime { + private ulong packedData; + + string toString() const { + char[64] buffer; + size_t pos; + + if(hasDate) { + pos += intToString(year, buffer[pos .. $], IntToStringArgs().withPadding(4)).length; + buffer[pos++] = '-'; + pos += intToString(month, buffer[pos .. $], IntToStringArgs().withPadding(2)).length; + buffer[pos++] = '-'; + pos += intToString(day, buffer[pos .. $], IntToStringArgs().withPadding(2)).length; + } + + if(hasTime) { + if(pos) + buffer[pos++] = 'T'; + + pos += intToString(hours, buffer[pos .. $], IntToStringArgs().withPadding(2)).length; + buffer[pos++] = ':'; + pos += intToString(minutes, buffer[pos .. $], IntToStringArgs().withPadding(2)).length; + buffer[pos++] = ':'; + pos += intToString(seconds, buffer[pos .. $], IntToStringArgs().withPadding(2)).length; + if(fractionalSeconds) { + buffer[pos++] = '.'; + pos += intToString(fractionalSeconds, buffer[pos .. $], IntToStringArgs().withPadding(4)).length; + } + } + + return buffer[0 .. pos].idup; + } + + /++ + +/ + int fractionalSeconds() const { return getFromMask(00, 10); } + /// ditto + void fractionalSeconds(int a) { setWithMask(a, 00, 10); } + + /// ditto + int seconds() const { return getFromMask(10, 6); } + /// ditto + void seconds(int a) { setWithMask(a, 10, 6); } + /// ditto + int minutes() const { return getFromMask(16, 6); } + /// ditto + void minutes(int a) { setWithMask(a, 16, 6); } + /// ditto + int hours() const { return getFromMask(22, 5); } + /// ditto + void hours(int a) { setWithMask(a, 22, 5); } + + /// ditto + int day() const { return getFromMask(27, 5); } + /// ditto + void day(int a) { setWithMask(a, 27, 5); } + /// ditto + int month() const { return getFromMask(32, 4); } + /// ditto + void month(int a) { setWithMask(a, 32, 4); } + /// ditto + int year() const { return getFromMask(36, 16); } + /// ditto + void year(int a) { setWithMask(a, 36, 16); } + + /// ditto + bool hasTime() const { return cast(bool) getFromMask(52, 1); } + /// ditto + void hasTime(bool a) { setWithMask(a, 52, 1); } + /// ditto + bool hasDate() const { return cast(bool) getFromMask(53, 1); } + /// ditto + void hasDate(bool a) { setWithMask(a, 53, 1); } + + private void setWithMask(int a, int bitOffset, int bitCount) { + auto mask = (1UL << bitCount) - 1; + + packedData &= ~(mask << bitOffset); + packedData |= (a & mask) << bitOffset; + } + + private int getFromMask(int bitOffset, int bitCount) const { + ulong packedData = this.packedData; + packedData >>= bitOffset; + + ulong mask = (1UL << bitCount) - 1; + + return cast(int) (packedData & mask); + } +} + +unittest { + PackedDateTime dt; + dt.hours = 14; + dt.minutes = 30; + dt.seconds = 25; + dt.hasTime = true; + + assert(dt.toString() == "14:30:25", dt.toString()); + + dt.hasTime = false; + dt.year = 2024; + dt.month = 5; + dt.day = 31; + dt.hasDate = true; + + assert(dt.toString() == "2024-05-31", dt.toString()); + dt.hasTime = true; + assert(dt.toString() == "2024-05-31T14:30:25", dt.toString()); +} + +/++ + Basically a Phobos SysTime but standing alone as a simple 6 4 bit integer (but wrapped) for compatibility with LimitedVariant. ++/ +struct SimplifiedUtcTimestamp { + long timestamp; + + string toString() const { + import core.stdc.time; + char[128] buffer; + auto ut = toUnixTime(); + tm* t = gmtime(&ut); + if(t is null) + return "null time"; + + return buffer[0 .. strftime(buffer.ptr, buffer.length, "%FT%H:%M:%SZ", t)].idup; + } + + version(Windows) + alias time_t = int; + + static SimplifiedUtcTimestamp fromUnixTime(time_t t) { + return SimplifiedUtcTimestamp(621_355_968_000_000_000L + t * 1_000_000_000L / 100); + } + + time_t toUnixTime() const { + return cast(time_t) ((timestamp - 621_355_968_000_000_000L) / 1_000_000_0); // hnsec = 7 digits + } +} + +unittest { + SimplifiedUtcTimestamp sut = SimplifiedUtcTimestamp.fromUnixTime(86_400); + assert(sut.toString() == "1970-01-02T00:00:00Z"); +} + +/++ + A limited variant to hold just a few types. It is made for the use of packing a small amount of extra data into error messages and some transit across virtual function boundaries. ++/ +/+ + ALL OF THESE ARE SUBJECT TO CHANGE + + * if length and ptr are both 0, it is null + * if ptr == 1, length is an integer + * if ptr == 2, length is an unsigned integer (suggest printing in hex) + * if ptr == 3, length is a combination of flags (suggest printing in binary) + * if ptr == 4, length is a unix permission thing (suggest printing in octal) + * if ptr == 5, length is a double float + * if ptr == 6, length is an Object ref (reinterpret casted to void*) + + * if ptr == 7, length is a ticks count (from MonoTime) + * if ptr == 8, length is a utc timestamp (hnsecs) + * if ptr == 9, length is a duration (signed hnsecs) + * if ptr == 10, length is a date or date time (bit packed, see flags in data to determine if it is a Date, Time, or DateTime) + * if ptr == 11, length is a dchar + * if ptr == 12, length is a bool (redundant to int?) + + 13, 14 reserved. prolly decimals. (4, 8 digits after decimal) + + * if ptr == 15, length must be 0. this holds an empty, non-null, SSO string. + * if ptr >= 16 && < 24, length is reinterpret-casted a small string of length of (ptr & 0x7) + 1 + + * if length == size_t.max, ptr is interpreted as a stringz + * if ptr >= 1024, it is a non-null D string or byte array. It is a string if the length high bit is clear, a byte array if it is set. the length is what is left after you mask that out. + + All other ptr values are reserved for future expansion. + + It basically can store: + null + type details = must be 0 + int (actually long) + type details = formatting hints + float (actually double) + type details = formatting hints + dchar (actually enum - upper half is the type tag, lower half is the member tag) + type details = ??? + decimal + type details = precision specifier + object + type details = ??? + timestamp + type details: ticks, utc timestamp, relative duration + + sso + stringz + + or it is bytes or a string; a normal D array (just bytes has a high bit set on length). + + But there are subtypes of some of those; ints can just have formatting hints attached. + Could reserve 0-7 as low level type flag (null, int, float, pointer, object) + 15-24 still can be the sso thing + + We have 10 bits really. + + 00000 00000 + ????? OOLLL + + The ????? are type details bits. + + 64 bits decmial to 4 points of precision needs... 14 bits for the small part (so max of 4 digits)? so 50 bits for the big part (max of about 1 quadrillion) + ...actually it can just be a dollars * 10000 + cents * 100. + ++/ +struct LimitedVariant { + + /++ + + +/ + enum Contains { + null_, + intDecimal, + intHex, + intBinary, + intOctal, + double_, + object, + + monoTime, + utcTimestamp, + duration, + dateTime, + + // FIXME boolean? char? decimal? + // could do enums by way of a pointer but kinda iffy + + // maybe some kind of prefixed string too for stuff like xml and json or enums etc. + + // fyi can also use stringzs or length-prefixed string pointers + emptySso, + stringSso, + stringz, + string, + bytes, + + invalid, + } + + /++ + Each datum stored in the LimitedVariant has a tag associated with it. + + Each tag belongs to one or more data families. + +/ + Contains contains() const { + auto tag = cast(size_t) ptr; + if(ptr is null && length is null) + return Contains.null_; + else switch(tag) { + case 1: return Contains.intDecimal; + case 2: return Contains.intHex; + case 3: return Contains.intBinary; + case 4: return Contains.intOctal; + case 5: return Contains.double_; + case 6: return Contains.object; + + case 7: return Contains.monoTime; + case 8: return Contains.utcTimestamp; + case 9: return Contains.duration; + case 10: return Contains.dateTime; + + case 15: return length is null ? Contains.emptySso : Contains.invalid; + default: + if(tag >= 16 && tag < 24) { + return Contains.stringSso; + } else if(tag >= 1024) { + if(cast(size_t) length == size_t.max) + return Contains.stringz; + else + return isHighBitSet ? Contains.bytes : Contains.string; + } else { + return Contains.invalid; + } + } + } + + /// ditto + bool containsNull() const { + return contains() == Contains.null_; + } + + /// ditto + bool containsInt() const { + with(Contains) + switch(contains) { + case intDecimal, intHex, intBinary, intOctal: + return true; + default: + return false; + } + } + + // all specializations of int... + + /// ditto + bool containsMonoTime() const { + return contains() == Contains.monoTime; + } + /// ditto + bool containsUtcTimestamp() const { + return contains() == Contains.utcTimestamp; + } + /// ditto + bool containsDuration() const { + return contains() == Contains.duration; + } + /// ditto + bool containsDateTime() const { + return contains() == Contains.dateTime; + } + + // done int specializations + + /// ditto + bool containsString() const { + with(Contains) + switch(contains) { + case null_, emptySso, stringSso, string: + case stringz: + return true; + default: + return false; + } + } + + /// ditto + bool containsDouble() const { + with(Contains) + switch(contains) { + case double_: + return true; + default: + return false; + } + } + + /// ditto + bool containsBytes() const { + with(Contains) + switch(contains) { + case bytes, null_: + return true; + default: + return false; + } + } + + private const(void)* length; + private const(ubyte)* ptr; + + private void Throw() const { + throw ArsdException!"LimitedVariant"(cast(size_t) length, cast(size_t) ptr); + } + + private bool isHighBitSet() const { + return (cast(size_t) length >> (size_t.sizeof * 8 - 1) & 0x1) != 0; + } + + /++ + getString gets a reference to the string stored internally, see [toString] to get a string representation or whatever is inside. + + +/ + const(char)[] getString() const return { + with(Contains) + switch(contains()) { + case null_: + return null; + case emptySso: + return (cast(const(char)*) ptr)[0 .. 0]; // zero length, non-null + case stringSso: + auto len = ((cast(size_t) ptr) & 0x7) + 1; + return (cast(char*) &length)[0 .. len]; + case string: + return (cast(const(char)*) ptr)[0 .. cast(size_t) length]; + case stringz: + return arsd.core.stringz(cast(char*) ptr).borrow; + default: + Throw(); assert(0); + } + } + + /// ditto + long getInt() const { + if(containsInt) + return cast(long) length; + else + Throw(); + assert(0); + } + + /// ditto + double getDouble() const { + if(containsDouble) { + floathack hack; + hack.e = cast(void*) length; // casting away const + return hack.d; + } else + Throw(); + assert(0); + } + + /// ditto + const(ubyte)[] getBytes() const { + with(Contains) + switch(contains()) { + case null_: + return null; + case bytes: + return ptr[0 .. (cast(size_t) length) & ((1UL << (size_t.sizeof * 8 - 1)) - 1)]; + default: + Throw(); assert(0); + } + } + + /// ditto + Object getObject() const { + with(Contains) + switch(contains()) { + case null_: + return null; + case object: + return cast(Object) length; // FIXME const correctness sigh + default: + Throw(); assert(0); + } + } + + /// ditto + MonoTime getMonoTime() const { + if(containsMonoTime) { + MonoTime time; + __traits(getMember, time, "_ticks") = cast(long) length; + return time; + } else + Throw(); + assert(0); + } + /// ditto + SimplifiedUtcTimestamp getUtcTimestamp() const { + if(containsUtcTimestamp) + return SimplifiedUtcTimestamp(cast(long) length); + else + Throw(); + assert(0); + } + /// ditto + Duration getDuration() const { + if(containsDuration) + return hnsecs(cast(long) length); + else + Throw(); + assert(0); + } + /// ditto + PackedDateTime getDateTime() const { + if(containsDateTime) + return PackedDateTime(cast(long) length); + else + Throw(); + assert(0); + } + + + /++ + + +/ + string toString() const { + + string intHelper(string prefix, int radix) { + char[128] buffer; + buffer[0 .. prefix.length] = prefix[]; + char[] toUse = buffer[prefix.length .. $]; + + auto got = intToString(getInt(), toUse[], IntToStringArgs().withRadix(radix)); + + return buffer[0 .. prefix.length + got.length].idup; + } + + with(Contains) + final switch(contains()) { + case null_: + return ""; + case intDecimal: + return intHelper("", 10); + case intHex: + return intHelper("0x", 16); + case intBinary: + return intHelper("0b", 2); + case intOctal: + return intHelper("0o", 8); + case emptySso, stringSso, string, stringz: + return getString().idup; + case bytes: + auto b = getBytes(); + + return ""; // FIXME + case object: + auto o = getObject(); + return o is null ? "null" : o.toString(); + case monoTime: + return getMonoTime.toString(); + case utcTimestamp: + return getUtcTimestamp().toString(); + case duration: + return getDuration().toString(); + case dateTime: + return getDateTime().toString(); + case double_: + auto d = getDouble(); + + import core.stdc.stdio; + char[128] buffer; + auto count = snprintf(buffer.ptr, buffer.length, "%.17lf", d); + return buffer[0 .. count].idup; + case invalid: + return ""; + } + } + + /++ + Note for integral types that are not `int` and `long` (for example, `short` or `ubyte`), you might want to explicitly convert them to `int`. + +/ + this(string s) { + ptr = cast(const(ubyte)*) s.ptr; + length = cast(void*) s.length; + } + + /// ditto + this(const(char)* stringz) { + if(stringz !is null) { + ptr = cast(const(ubyte)*) stringz; + length = cast(void*) size_t.max; + } else { + ptr = null; + length = null; + } + } + + /// ditto + this(const(ubyte)[] b) { + ptr = cast(const(ubyte)*) b.ptr; + length = cast(void*) (b.length | (1UL << (size_t.sizeof * 8 - 1))); + } + + /// ditto + this(long l, int base = 10) { + int tag; + switch(base) { + case 10: tag = 1; break; + case 16: tag = 2; break; + case 2: tag = 3; break; + case 8: tag = 4; break; + default: assert(0, "You passed an invalid base to LimitedVariant"); + } + ptr = cast(ubyte*) tag; + length = cast(void*) l; + } + + /// ditto + this(int i, int base = 10) { + this(cast(long) i, base); + } + + /// ditto + this(bool i) { + // FIXME? + this(cast(long) i); + } + + /// ditto + this(double d) { + // the reinterpret cast hack crashes dmd! omg + ptr = cast(ubyte*) 5; + + floathack h; + h.d = d; + + this.length = h.e; + } + + /// ditto + this(Object o) { + this.ptr = cast(ubyte*) 6; + this.length = cast(void*) o; + } + + /// ditto + this(MonoTime a) { + this.ptr = cast(ubyte*) 7; + this.length = cast(void*) a.ticks; + } + + /// ditto + this(SimplifiedUtcTimestamp a) { + this.ptr = cast(ubyte*) 8; + this.length = cast(void*) a.timestamp; + } + + /// ditto + this(Duration a) { + this.ptr = cast(ubyte*) 9; + this.length = cast(void*) a.total!"hnsecs"; + } + + /// ditto + this(PackedDateTime a) { + this.ptr = cast(ubyte*) 10; + this.length = cast(void*) a.packedData; + } +} + +unittest { + LimitedVariant v = LimitedVariant("foo"); + assert(v.containsString()); + assert(!v.containsInt()); + assert(v.getString() == "foo"); + + LimitedVariant v2 = LimitedVariant(4); + assert(v2.containsInt()); + assert(!v2.containsString()); + assert(v2.getInt() == 4); + + LimitedVariant v3 = LimitedVariant(cast(ubyte[]) [1, 2, 3]); + assert(v3.containsBytes()); + assert(!v3.containsString()); + assert(v3.getBytes() == [1, 2, 3]); +} + +private union floathack { + // in 32 bit we'll use float instead since it at least fits in the void* + static if(double.sizeof == (void*).sizeof) { + double d; + } else { + float d; + } + void* e; +} + +/++ + This is a dummy type to indicate the end of normal arguments and the beginning of the file/line inferred args. It is meant to ensure you don't accidentally send a string that is interpreted as a filename when it was meant to be a normal argument to the function and trigger the wrong overload. ++/ +struct ArgSentinel {} + +/++ + A trivial wrapper around C's malloc that creates a D slice. It multiples n by T.sizeof and returns the slice of the pointer from 0 to n. + + Please note that the ptr might be null - it is your responsibility to check that, same as normal malloc. Check `ret is null` specifically, since `ret.length` will always be `n`, even if the `malloc` failed. + + Remember to `free` the returned pointer with `core.stdc.stdlib.free(ret.ptr);` + + $(TIP + I strongly recommend you simply use the normal garbage collector unless you have a very specific reason not to. + ) + + See_Also: + [mallocedStringz] ++/ +T[] mallocSlice(T)(size_t n) { + import c = core.stdc.stdlib; + + return (cast(T*) c.malloc(n * T.sizeof))[0 .. n]; +} + +/++ + Uses C's malloc to allocate a copy of `original` with an attached zero terminator. It may return a slice with a `null` pointer (but non-zero length!) if `malloc` fails and you are responsible for freeing the returned pointer with `core.stdc.stdlib.free(ret.ptr)`. + + $(TIP + I strongly recommend you use [CharzBuffer] or Phobos' [std.string.toStringz] instead unless there's a special reason not to. + ) + + See_Also: + [CharzBuffer] for a generally better alternative. You should only use `mallocedStringz` where `CharzBuffer` cannot be used (e.g. when druntime is not usable or you have no stack space for the temporary buffer). + + [mallocSlice] is the function this function calls, so the notes in its documentation applies here too. ++/ +char[] mallocedStringz(in char[] original) { + auto slice = mallocSlice!char(original.length + 1); + if(slice is null) + return null; + slice[0 .. original.length] = original[]; + slice[original.length] = 0; + return slice; +} + +/++ + Basically a `scope class` you can return from a function or embed in another aggregate. ++/ +struct OwnedClass(Class) { + ubyte[__traits(classInstanceSize, Class)] rawData; + + static OwnedClass!Class defaultConstructed() { + OwnedClass!Class i = OwnedClass!Class.init; + i.initializeRawData(); + return i; + } + + private void initializeRawData() @trusted { + if(!this) + rawData[] = cast(ubyte[]) typeid(Class).initializer[]; + } + + this(T...)(T t) { + initializeRawData(); + rawInstance.__ctor(t); + } + + bool opCast(T : bool)() @trusted { + return !(*(cast(void**) rawData.ptr) is null); + } + + @disable this(); + @disable this(this); + + Class rawInstance() return @trusted { + if(!this) + throw new Exception("null"); + return cast(Class) rawData.ptr; + } + + alias rawInstance this; + + ~this() @trusted { + if(this) + .destroy(rawInstance()); + } +} + +// might move RecyclableMemory here + +version(Posix) +package(arsd) void makeNonBlocking(int fd) { + import core.sys.posix.fcntl; + auto flags = fcntl(fd, F_GETFL, 0); + if(flags == -1) + throw new ErrnoApiException("fcntl get", errno); + flags |= O_NONBLOCK; + auto s = fcntl(fd, F_SETFL, flags); + if(s == -1) + throw new ErrnoApiException("fcntl set", errno); +} + +version(Posix) +package(arsd) void setCloExec(int fd) { + import core.sys.posix.fcntl; + auto flags = fcntl(fd, F_GETFD, 0); + if(flags == -1) + throw new ErrnoApiException("fcntl get", errno); + flags |= FD_CLOEXEC; + auto s = fcntl(fd, F_SETFD, flags); + if(s == -1) + throw new ErrnoApiException("fcntl set", errno); +} + + +/++ + A helper object for temporarily constructing a string appropriate for the Windows API from a D UTF-8 string. + + + It will use a small internal static buffer is possible, and allocate a new buffer if the string is too big. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +struct WCharzBuffer { + private wchar[] buffer; + private wchar[128] staticBuffer = void; + + /// Length of the string, excluding the zero terminator. + size_t length() { + return buffer.length; + } + + // Returns the pointer to the internal buffer. You must assume its lifetime is less than that of the WCharzBuffer. It is zero-terminated. + wchar* ptr() { + return buffer.ptr; + } + + /// Returns the slice of the internal buffer, excluding the zero terminator (though there is one present right off the end of the slice). You must assume its lifetime is less than that of the WCharzBuffer. + wchar[] slice() { + return buffer; + } + + /// Copies it into a static array of wchars + void copyInto(R)(ref R r) { + static if(is(R == wchar[N], size_t N)) { + r[0 .. this.length] = slice[]; + r[this.length] = 0; + } else static assert(0, "can only copy into wchar[n], not " ~ R.stringof); + } + + /++ + conversionFlags = [WindowsStringConversionFlags] + +/ + this(in char[] data, int conversionFlags = 0) { + conversionFlags |= WindowsStringConversionFlags.zeroTerminate; // this ALWAYS zero terminates cuz of its name + auto sz = sizeOfConvertedWstring(data, conversionFlags); + if(sz > staticBuffer.length) + buffer = new wchar[](sz); + else + buffer = staticBuffer[]; + + buffer = makeWindowsString(data, buffer, conversionFlags); + } +} + +/++ + Alternative for toStringz + + History: + Added March 18, 2023 (dub v11.0) ++/ +struct CharzBuffer { + private char[] buffer; + private char[128] staticBuffer = void; + + /// Length of the string, excluding the zero terminator. + size_t length() { + assert(buffer.length > 0); + return buffer.length - 1; + } + + // Returns the pointer to the internal buffer. You must assume its lifetime is less than that of the CharzBuffer. It is zero-terminated. + char* ptr() { + return buffer.ptr; + } + + /// Returns the slice of the internal buffer, excluding the zero terminator (though there is one present right off the end of the slice). You must assume its lifetime is less than that of the CharzBuffer. + char[] slice() { + assert(buffer.length > 0); + return buffer[0 .. $-1]; + } + + /// Copies it into a static array of chars + void copyInto(R)(ref R r) { + static if(is(R == char[N], size_t N)) { + r[0 .. this.length] = slice[]; + r[this.length] = 0; + } else static assert(0, "can only copy into char[n], not " ~ R.stringof); + } + + @disable this(); + @disable this(this); + + /++ + Copies `data` into the CharzBuffer, allocating a new one if needed, and zero-terminates it. + +/ + this(in char[] data) { + if(data.length + 1 > staticBuffer.length) + buffer = new char[](data.length + 1); + else + buffer = staticBuffer[]; + + buffer[0 .. data.length] = data[]; + buffer[data.length] = 0; + } +} + +/++ + Given the string `str`, converts it to a string compatible with the Windows API and puts the result in `buffer`, returning the slice of `buffer` actually used. `buffer` must be at least [sizeOfConvertedWstring] elements long. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +wchar[] makeWindowsString(in char[] str, wchar[] buffer, int conversionFlags = WindowsStringConversionFlags.zeroTerminate) { + if(str.length == 0) + return null; + + int pos = 0; + dchar last; + foreach(dchar c; str) { + if(c <= 0xFFFF) { + if((conversionFlags & WindowsStringConversionFlags.convertNewLines) && c == 10 && last != 13) + buffer[pos++] = 13; + buffer[pos++] = cast(wchar) c; + } else if(c <= 0x10FFFF) { + buffer[pos++] = cast(wchar)((((c - 0x10000) >> 10) & 0x3FF) + 0xD800); + buffer[pos++] = cast(wchar)(((c - 0x10000) & 0x3FF) + 0xDC00); + } + + last = c; + } + + if(conversionFlags & WindowsStringConversionFlags.zeroTerminate) { + buffer[pos] = 0; + } + + return buffer[0 .. pos]; +} + +/++ + Converts the Windows API string `str` to a D UTF-8 string, storing it in `buffer`. Returns the slice of `buffer` actually used. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +char[] makeUtf8StringFromWindowsString(in wchar[] str, char[] buffer) { + if(str.length == 0) + return null; + + auto got = WideCharToMultiByte(CP_UTF8, 0, str.ptr, cast(int) str.length, buffer.ptr, cast(int) buffer.length, null, null); + if(got == 0) { + if(GetLastError() == ERROR_INSUFFICIENT_BUFFER) + throw new object.Exception("not enough buffer"); + else + throw new object.Exception("conversion"); // FIXME: GetLastError + } + return buffer[0 .. got]; +} + +/++ + Converts the Windows API string `str` to a newly-allocated D UTF-8 string. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +string makeUtf8StringFromWindowsString(in wchar[] str) { + char[] buffer; + auto got = WideCharToMultiByte(CP_UTF8, 0, str.ptr, cast(int) str.length, null, 0, null, null); + buffer.length = got; + + // it is unique because we just allocated it above! + return cast(string) makeUtf8StringFromWindowsString(str, buffer); +} + +/// ditto +version(Windows) +string makeUtf8StringFromWindowsString(wchar* str) { + char[] buffer; + auto got = WideCharToMultiByte(CP_UTF8, 0, str, -1, null, 0, null, null); + buffer.length = got; + + got = WideCharToMultiByte(CP_UTF8, 0, str, -1, buffer.ptr, cast(int) buffer.length, null, null); + if(got == 0) { + if(GetLastError() == ERROR_INSUFFICIENT_BUFFER) + throw new object.Exception("not enough buffer"); + else + throw new object.Exception("conversion"); // FIXME: GetLastError + } + return cast(string) buffer[0 .. got]; +} + +// only used from minigui rn +package int findIndexOfZero(in wchar[] str) { + foreach(idx, wchar ch; str) + if(ch == 0) + return cast(int) idx; + return cast(int) str.length; +} +package int findIndexOfZero(in char[] str) { + foreach(idx, char ch; str) + if(ch == 0) + return cast(int) idx; + return cast(int) str.length; +} + +/++ + Returns a minimum buffer length to hold the string `s` with the given conversions. It might be slightly larger than necessary, but is guaranteed to be big enough to hold it. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +int sizeOfConvertedWstring(in char[] s, int conversionFlags) { + int size = 0; + + if(conversionFlags & WindowsStringConversionFlags.convertNewLines) { + // need to convert line endings, which means the length will get bigger. + + // BTW I betcha this could be faster with some simd stuff. + char last; + foreach(char ch; s) { + if(ch == 10 && last != 13) + size++; // will add a 13 before it... + size++; + last = ch; + } + } else { + // no conversion necessary, just estimate based on length + /* + I don't think there's any string with a longer length + in code units when encoded in UTF-16 than it has in UTF-8. + This will probably over allocate, but that's OK. + */ + size = cast(int) s.length; + } + + if(conversionFlags & WindowsStringConversionFlags.zeroTerminate) + size++; + + return size; +} + +/++ + Used by [makeWindowsString] and [WCharzBuffer] + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +version(Windows) +enum WindowsStringConversionFlags : int { + /++ + Append a zero terminator to the string. + +/ + zeroTerminate = 1, + /++ + Converts newlines from \n to \r\n. + +/ + convertNewLines = 2, +} + +/++ + An int printing function that doesn't need to import Phobos. Can do some of the things std.conv.to and std.format.format do. + + The buffer must be sized to hold the converted number. 32 chars is enough for most anything. + + Returns: the slice of `buffer` containing the converted number. ++/ +char[] intToString(long value, char[] buffer, IntToStringArgs args = IntToStringArgs.init) { + const int radix = args.radix ? args.radix : 10; + const int digitsPad = args.padTo; + const int groupSize = args.groupSize; + + int pos; + + if(value < 0) { + buffer[pos++] = '-'; + value = -value; + } + + int start = pos; + int digitCount; + + do { + auto remainder = value % radix; + value = value / radix; + + buffer[pos++] = cast(char) (remainder < 10 ? (remainder + '0') : (remainder - 10 + args.ten)); + digitCount++; + } while(value); + + if(digitsPad > 0) { + while(digitCount < digitsPad) { + buffer[pos++] = args.padWith; + digitCount++; + } + } + + assert(pos >= 1); + assert(pos - start > 0); + + auto reverseSlice = buffer[start .. pos]; + for(int i = 0; i < reverseSlice.length / 2; i++) { + auto paired = cast(int) reverseSlice.length - i - 1; + char tmp = reverseSlice[i]; + reverseSlice[i] = reverseSlice[paired]; + reverseSlice[paired] = tmp; + } + + return buffer[0 .. pos]; +} + +/// ditto +struct IntToStringArgs { + private { + ubyte padTo; + char padWith; + ubyte radix; + char ten; + ubyte groupSize; + char separator; + } + + IntToStringArgs withPadding(int padTo, char padWith = '0') { + IntToStringArgs args = this; + args.padTo = cast(ubyte) padTo; + args.padWith = padWith; + return args; + } + + IntToStringArgs withRadix(int radix, char ten = 'a') { + IntToStringArgs args = this; + args.radix = cast(ubyte) radix; + args.ten = ten; + return args; + } + + IntToStringArgs withGroupSeparator(int groupSize, char separator = '_') { + IntToStringArgs args = this; + args.groupSize = cast(ubyte) groupSize; + args.separator = separator; + return args; + } +} + +unittest { + char[32] buffer; + assert(intToString(0, buffer[]) == "0"); + assert(intToString(-1, buffer[]) == "-1"); + assert(intToString(-132, buffer[]) == "-132"); + assert(intToString(-1932, buffer[]) == "-1932"); + assert(intToString(1, buffer[]) == "1"); + assert(intToString(132, buffer[]) == "132"); + assert(intToString(1932, buffer[]) == "1932"); + + assert(intToString(0x1, buffer[], IntToStringArgs().withRadix(16)) == "1"); + assert(intToString(0x1b, buffer[], IntToStringArgs().withRadix(16)) == "1b"); + assert(intToString(0xef1, buffer[], IntToStringArgs().withRadix(16)) == "ef1"); + + assert(intToString(0xef1, buffer[], IntToStringArgs().withRadix(16).withPadding(8)) == "00000ef1"); + assert(intToString(-0xef1, buffer[], IntToStringArgs().withRadix(16).withPadding(8)) == "-00000ef1"); + assert(intToString(-0xef1, buffer[], IntToStringArgs().withRadix(16, 'A').withPadding(8, ' ')) == "- EF1"); +} + +/++ + History: + Moved from color.d to core.d in March 2023 (dub v11.0). ++/ +nothrow @safe @nogc pure +inout(char)[] stripInternal(return inout(char)[] s) { + bool isAllWhitespace = true; + foreach(i, char c; s) + if(c != ' ' && c != '\t' && c != '\n' && c != '\r') { + s = s[i .. $]; + isAllWhitespace = false; + break; + } + + if(isAllWhitespace) + return s[$..$]; + + for(int a = cast(int)(s.length - 1); a > 0; a--) { + char c = s[a]; + if(c != ' ' && c != '\t' && c != '\n' && c != '\r') { + s = s[0 .. a + 1]; + break; + } + } + + return s; +} + +/// ditto +nothrow @safe @nogc pure +inout(char)[] stripRightInternal(return inout(char)[] s) { + bool isAllWhitespace = true; + foreach_reverse(a, c; s) { + if(c != ' ' && c != '\t' && c != '\n' && c != '\r') { + s = s[0 .. a + 1]; + isAllWhitespace = false; + break; + } + } + if(isAllWhitespace) + s = s[0..0]; + + return s; + +} + +/++ + Shortcut for converting some types to string without invoking Phobos (but it will as a last resort). + + History: + Moved from color.d to core.d in March 2023 (dub v11.0). ++/ +string toStringInternal(T)(T t) { + char[32] buffer; + static if(is(T : string)) + return t; + else static if(is(T : long)) + return intToString(t, buffer[]).idup; + else static if(is(T == enum)) { + switch(t) { + foreach(memberName; __traits(allMembers, T)) { + case __traits(getMember, T, memberName): + return memberName; + } + default: + return ""; + } + } else { + import std.conv; + return to!string(t); + } +} + +/++ + ++/ +string flagsToString(Flags)(ulong value) { + string r; + + void add(string memberName) { + if(r.length) + r ~= " | "; + r ~= memberName; + } + + string none = ""; + + foreach(memberName; __traits(allMembers, Flags)) { + auto flag = cast(ulong) __traits(getMember, Flags, memberName); + if(flag) { + if((value & flag) == flag) + add(memberName); + } else { + none = memberName; + } + } + + if(r.length == 0) + r = none; + + return r; +} + +unittest { + enum MyFlags { + none = 0, + a = 1, + b = 2 + } + + assert(flagsToString!MyFlags(3) == "a | b"); + assert(flagsToString!MyFlags(0) == "none"); + assert(flagsToString!MyFlags(2) == "b"); +} + +// technically s is octets but meh +package string encodeUriComponent(string s) { + char[3] encodeChar(char c) { + char[3] buffer; + buffer[0] = '%'; + + enum hexchars = "0123456789ABCDEF"; + buffer[1] = hexchars[c >> 4]; + buffer[2] = hexchars[c & 0x0f]; + + return buffer; + } + + string n; + size_t previous = 0; + foreach(idx, char ch; s) { + if( + (ch >= 'A' && ch <= 'Z') + || + (ch >= 'a' && ch <= 'z') + || + (ch >= '0' && ch <= '9') + || ch == '-' || ch == '_' || ch == '.' || ch == '~' // unreserved set + || ch == '!' || ch == '*' || ch == '\''|| ch == '(' || ch == ')' // subdelims but allowed in uri component (phobos also no encode them) + ) { + // does not need encoding + } else { + n ~= s[previous .. idx]; + n ~= encodeChar(ch); + previous = idx + 1; + } + } + + if(n.length) { + n ~= s[previous .. $]; + return n; + } else { + return s; // nothing needed encoding + } +} +unittest { + assert(encodeUriComponent("foo") == "foo"); + assert(encodeUriComponent("f33Ao") == "f33Ao"); + assert(encodeUriComponent("/") == "%2F"); + assert(encodeUriComponent("/foo") == "%2Ffoo"); + assert(encodeUriComponent("foo/") == "foo%2F"); + assert(encodeUriComponent("foo/bar") == "foo%2Fbar"); + assert(encodeUriComponent("foo/bar/") == "foo%2Fbar%2F"); +} + +// FIXME: I think if translatePlusToSpace we're supposed to do newline normalization too +package string decodeUriComponent(string s, bool translatePlusToSpace = false) { + int skipping = 0; + size_t previous = 0; + string n = null; + foreach(idx, char ch; s) { + if(skipping) { + skipping--; + continue; + } + + if(ch == '%') { + int hexDecode(char c) { + if(c >= 'A' && c <= 'F') + return c - 'A' + 10; + else if(c >= 'a' && c <= 'f') + return c - 'a' + 10; + else if(c >= '0' && c <= '9') + return c - '0' + 0; + else + throw ArsdException!"Invalid percent-encoding"("Invalid char encountered", idx, s); + } + + skipping = 2; + n ~= s[previous .. idx]; + + if(idx + 2 >= s.length) + throw ArsdException!"Invalid percent-encoding"("End of string reached", idx, s); + + n ~= (hexDecode(s[idx + 1]) << 4) | hexDecode(s[idx + 2]); + + previous = idx + 3; + } else if(translatePlusToSpace && ch == '+') { + n ~= s[previous .. idx]; + n ~= " "; + previous = idx + 1; + } + } + + if(n.length) { + n ~= s[previous .. $]; + return n; + } else { + return s; // nothing needed decoding + } +} + +unittest { + assert(decodeUriComponent("foo") == "foo"); + assert(decodeUriComponent("%2F") == "/"); + assert(decodeUriComponent("%2f") == "/"); + assert(decodeUriComponent("%2Ffoo") == "/foo"); + assert(decodeUriComponent("foo%2F") == "foo/"); + assert(decodeUriComponent("foo%2Fbar") == "foo/bar"); + assert(decodeUriComponent("foo%2Fbar%2F") == "foo/bar/"); + assert(decodeUriComponent("%2F%2F%2F") == "///"); + + assert(decodeUriComponent("+") == "+"); + assert(decodeUriComponent("+", true) == " "); +} + +private auto toDelegate(T)(T t) { + // static assert(is(T == function)); // lol idk how to do what i actually want here + + static if(is(T Return == return)) + static if(is(typeof(*T) Params == __parameters)) { + static struct Wrapper { + Return call(Params params) { + return (cast(T) &this)(params); + } + } + return &((cast(Wrapper*) t).call); + } else static assert(0, "could not get params"); + else static assert(0, "could not get return value"); +} + +unittest { + int function(int) fn; + fn = (a) { return a; }; + + int delegate(int) dg = toDelegate(fn); + + assert(dg.ptr is fn); // it stores the original function as the context pointer + assert(dg.funcptr !is fn); // which is called through a lil trampoline + assert(dg(5) == 5); // and forwards the args correctly +} + +/++ + This populates a struct from a list of values (or other expressions, but it only looks at the values) based on types of the members, with one exception: `bool` members.. maybe. + + It is intended for collecting a record of relevant UDAs off a symbol in a single call like this: + + --- + struct Name { + string n; + } + + struct Validator { + string regex; + } + + struct FormInfo { + Name name; + Validator validator; + } + + @Name("foo") @Validator(".*") + void foo() {} + + auto info = populateFromUdas!(FormInfo, __traits(getAttributes, foo)); + assert(info.name == Name("foo")); + assert(info.validator == Validator(".*")); + --- + + Note that instead of UDAs, you can also pass a variadic argument list and get the same result, but the function is `populateFromArgs` and you pass them as the runtime list to bypass "args cannot be evaluated at compile time" errors: + + --- + void foo(T...)(T t) { + auto info = populateFromArgs!(FormInfo)(t); + // assuming the call below + assert(info.name == Name("foo")); + assert(info.validator == Validator(".*")); + } + + foo(Name("foo"), Validator(".*")); + --- + + The benefit of this over constructing the struct directly is that the arguments can be reordered or missing. Its value is diminished with named arguments in the language. ++/ +template populateFromUdas(Struct, UDAs...) { + enum Struct populateFromUdas = () { + Struct ret; + foreach(memberName; __traits(allMembers, Struct)) { + alias memberType = typeof(__traits(getMember, Struct, memberName)); + foreach(uda; UDAs) { + static if(is(memberType == PresenceOf!a, a)) { + static if(__traits(isSame, a, uda)) + __traits(getMember, ret, memberName) = true; + } + else + static if(is(typeof(uda) : memberType)) { + __traits(getMember, ret, memberName) = uda; + } + } + } + + return ret; + }(); +} + +/// ditto +Struct populateFromArgs(Struct, Args...)(Args args) { + Struct ret; + foreach(memberName; __traits(allMembers, Struct)) { + alias memberType = typeof(__traits(getMember, Struct, memberName)); + foreach(arg; args) { + static if(is(typeof(arg == memberType))) { + __traits(getMember, ret, memberName) = arg; + } + } + } + + return ret; +} + +/// ditto +struct PresenceOf(alias a) { + bool there; + alias there this; +} + +/// +unittest { + enum a; + enum b; + struct Name { string name; } + struct Info { + Name n; + PresenceOf!a athere; + PresenceOf!b bthere; + int c; + } + + void test() @a @Name("test") {} + + auto info = populateFromUdas!(Info, __traits(getAttributes, test)); + assert(info.n == Name("test")); // but present ones are in there + assert(info.athere == true); // non-values can be tested with PresenceOf!it, which works like a bool + assert(info.bthere == false); + assert(info.c == 0); // absent thing will keep the default value +} + +/++ + Declares a delegate property with several setters to allow for handlers that don't care about the arguments. + + Throughout the arsd library, you will often see types of these to indicate that you can set listeners with or without arguments. If you care about the details of the callback event, you can set a delegate that declares them. And if you don't, you can set one that doesn't even declare them and it will be ignored. ++/ +struct FlexibleDelegate(DelegateType) { + // please note that Parameters and ReturnType are public now! + static if(is(DelegateType FunctionType == delegate)) + static if(is(FunctionType Parameters == __parameters)) + static if(is(DelegateType ReturnType == return)) { + + /++ + Calls the currently set delegate. + + Diagnostics: + If the callback delegate has not been set, this may cause a null pointer dereference. + +/ + ReturnType opCall(Parameters args) { + return dg(args); + } + + /++ + Use `if(thing)` to check if the delegate is null or not. + +/ + bool opCast(T : bool)() { + return dg !is null; + } + + /++ + These opAssign overloads are what puts the flexibility in the flexible delegate. + + Bugs: + The other overloads do not keep attributes like `nothrow` on the `dg` parameter, making them unusable if `DelegateType` requires them. I consider the attributes more trouble than they're worth anyway, and the language's poor support for composing them doesn't help any. I have no need for them and thus no plans to add them in the overloads at this time. + +/ + void opAssign(DelegateType dg) { + this.dg = dg; + } + + /// ditto + void opAssign(ReturnType delegate() dg) { + this.dg = (Parameters ignored) => dg(); + } + + /// ditto + void opAssign(ReturnType function(Parameters params) dg) { + this.dg = (Parameters params) => dg(params); + } + + /// ditto + void opAssign(ReturnType function() dg) { + this.dg = (Parameters ignored) => dg(); + } + + /// ditto + void opAssign(typeof(null) explicitNull) { + this.dg = null; + } + + private DelegateType dg; + } + else static assert(0, DelegateType.stringof ~ " failed return value check"); + else static assert(0, DelegateType.stringof ~ " failed parameters check"); + else static assert(0, DelegateType.stringof ~ " failed delegate check"); +} + +/++ + ++/ +unittest { + // you don't have to put the arguments in a struct, but i recommend + // you do as it is more future proof - you can add more info to the + // struct without breaking user code that consumes it. + struct MyEventArguments { + + } + + // then you declare it just adding FlexibleDelegate!() around the + // plain delegate type you'd normally use + FlexibleDelegate!(void delegate(MyEventArguments args)) callback; + + // until you set it, it will be null and thus be false in any boolean check + assert(!callback); + + // can set it to the properly typed thing + callback = delegate(MyEventArguments args) {}; + + // and now it is no longer null + assert(callback); + + // or if you don't care about the args, you can leave them off + callback = () {}; + + // and it works if the compiler types you as a function instead of delegate too + // (which happens automatically if you don't access any local state or if you + // explicitly define it as a function) + + callback = function(MyEventArguments args) { }; + + // can set it back to null explicitly if you ever wanted + callback = null; + + // the reflection info used internally also happens to be exposed publicly + // which can actually sometimes be nice so if the language changes, i'll change + // the code to keep this working. + static assert(is(callback.ReturnType == void)); + + // which can be convenient if the params is an annoying type since you can + // consistently use something like this too + callback = (callback.Parameters params) {}; + + // check for null and call it pretty normally + if(callback) + callback(MyEventArguments()); +} + +/+ + ====================== + ERROR HANDLING HELPERS + ====================== ++/ + +/+ + + arsd code shouldn't be using Exception. Really, I don't think any code should be - instead, construct an appropriate object with structured information. + + If you want to catch someone else's Exception, use `catch(object.Exception e)`. ++/ +//package deprecated struct Exception {} + + +/++ + Base class representing my exceptions. You should almost never work with this directly, but you might catch it as a generic thing. Catch it before generic `object.Exception` or `object.Throwable` in any catch chains. + + + $(H3 General guidelines for exceptions) + + The purpose of an exception is to cancel a task that has proven to be impossible and give the programmer enough information to use at a higher level to decide what to do about it. + + Cancelling a task is accomplished with the `throw` keyword. The transmission of information to a higher level is done by the language runtime. The decision point is marked by the `catch` keyword. The part missing - the job of the `Exception` class you construct and throw - is to gather the information that will be useful at a later decision point. + + It is thus important that you gather as much useful information as possible and keep it in a way that the code catching the exception can still interpret it when constructing an exception. Other concerns are secondary to this to this primary goal. + + With this in mind, here's some guidelines for exception handling in arsd code. + + $(H4 Allocations and lifetimes) + + Don't get clever with exception allocations. You don't know what the catcher is going to do with an exception and you don't want the error handling scheme to introduce its own tricky bugs. Remember, an exception object's first job is to deliver useful information up the call chain in a way this code can use it. You don't know what this code is or what it is going to do. + + Keep your memory management schemes simple and let the garbage collector do its job. + + $(LIST + * All thrown exceptions should be allocated with the `new` keyword. + + * Members inside the exception should be value types or have infinite lifetime (that is, be GC managed). + + * While this document is concerned with throwing, you might want to add additional information to an in-flight exception, and this is done by catching, so you need to know how that works too, and there is a global compiler switch that can change things, so even inside arsd we can't completely avoid its implications. + + DIP1008's presence complicates things a bit on the catch side - if you catch an exception and return it from a function, remember to `ex.refcount = ex.refcount + 1;` so you don't introduce more use-after-free woes for those unfortunate souls. + ) + + $(H4 Error strings) + + Strings can deliver useful information to people reading the message, but are often suboptimal for delivering useful information to other chunks of code. Remember, an exception's first job is to be caught by another block of code. Printing to users is a last resort; even if you want a user-readable error message, an exception is not the ideal way to deliver one since it is constructed in the guts of a failed task, without the higher level context of what the user was actually trying to do. User error messages ought to be made from information in the exception, combined with higher level knowledge. This is best done in a `catch` block, not a `throw` statement. + + As such, I recommend that you: + + $(LIST + * Don't concatenate error strings at the throw site. Instead, pass the data you would have used to build the string as actual data to the constructor. This lets catchers see the original data without having to try to extract it from a string. For unique data, you will likely need a unique exception type. More on this in the next section. + + * Don't construct error strings in a constructor either, for the same reason. Pass the useful data up the call chain, as exception members, to the maximum extent possible. Exception: if you are passed some data with a temporary lifetime that is important enough to pass up the chain. You may `.idup` or `to!string` to preserve as much data as you can before it is lost, but still store it in a separate member of the Exception subclass object. + + * $(I Do) construct strings out of public members in [getAdditionalPrintableInformation]. When this is called, the user has requested as much relevant information as reasonable in string format. Still, avoid concatenation - it lets you pass as many key/value pairs as you like to the caller. They can concatenate as needed. However, note the words "public members" - everything you do in `getAdditionalPrintableInformation` ought to also be possible for code that caught your exception via your public methods and properties. + ) + + $(H4 Subclasses) + + Any exception with unique data types should be a unique class. Whenever practical, this should be one you write and document at the top-level of a module. But I know we get lazy - me too - and this is why in standard D we'd often fall back to `throw new Exception("some string " ~ some info)`. To help resist these urges, I offer some helper functions to use instead that better achieve the key goal of exceptions - passing structured data up a call chain - while still being convenient to write. + + See: [ArsdException], [Win32Enforce] + ++/ +class ArsdExceptionBase : object.Exception { + /++ + Don't call this except from other exceptions; this is essentially an abstract class. + + Params: + operation = the specific operation that failed, throwing the exception + +/ + package this(string operation, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(operation, file, line, next); + } + + /++ + The toString method will print out several components: + + $(LIST + * The file, line, static message, and object class name from the constructor. You can access these independently with the members `file`, `line`, `msg`, and [printableExceptionName]. + * The generic category codes stored with this exception + * Additional members stored with the exception child classes (e.g. platform error codes, associated function arguments) + * The stack trace associated with the exception. You can access these lines independently with `foreach` over the `info` member. + ) + + This is meant to be read by the developer, not end users. You should wrap your user-relevant tasks in a try/catch block and construct more appropriate error messages from context available there, using the individual properties of the exception to add richness. + +/ + final override void toString(scope void delegate(in char[]) sink) const { + // class name and info from constructor + sink(printableExceptionName); + sink("@"); + sink(file); + sink("("); + char[16] buffer; + sink(intToString(line, buffer[])); + sink("): "); + sink(message); + + getAdditionalPrintableInformation((string name, in char[] value) { + sink("\n"); + sink(name); + sink(": "); + sink(value); + }); + + // full stack trace + sink("\n----------------\n"); + foreach(str; info) { + sink(str); + sink("\n"); + } + } + /// ditto + final override string toString() { + string s; + toString((in char[] chunk) { s ~= chunk; }); + return s; + } + + /++ + Users might like to see additional information with the exception. API consumers should pull this out of properties on your child class, but the parent class might not be able to deal with the arbitrary types at runtime the children can introduce, so bringing them all down to strings simplifies that. + + Overrides should always call `super.getAdditionalPrintableInformation(sink);` before adding additional information by calling the sink with other arguments afterward. + + You should spare no expense in preparing this information - translate error codes, build rich strings, whatever it takes - to make the information here useful to the reader. + +/ + void getAdditionalPrintableInformation(scope void delegate(string name, in char[] value) sink) const { + + } + + /++ + This is the name of the exception class, suitable for printing. This should be static data (e.g. a string literal). Override it in subclasses. + +/ + string printableExceptionName() const { + return typeid(this).name; + } + + /// deliberately hiding `Throwable.msg`. Use [message] and [toString] instead. + @disable final void msg() {} + + override const(char)[] message() const { + return super.msg; + } +} + +/++ + ++/ +class InvalidArgumentsException : ArsdExceptionBase { + static struct InvalidArgument { + string name; + string description; + LimitedVariant givenValue; + } + + InvalidArgument[] invalidArguments; + + this(InvalidArgument[] invalidArguments, string functionName = __PRETTY_FUNCTION__, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.invalidArguments = invalidArguments; + super(functionName, file, line, next); + } + + this(string argumentName, string argumentDescription, LimitedVariant givenArgumentValue = LimitedVariant.init, string functionName = __PRETTY_FUNCTION__, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this([ + InvalidArgument(argumentName, argumentDescription, givenArgumentValue) + ], functionName, file, line, next); + } + + this(string argumentName, string argumentDescription, string functionName = __PRETTY_FUNCTION__, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this(argumentName, argumentDescription, LimitedVariant.init, functionName, file, line, next); + } + + override void getAdditionalPrintableInformation(scope void delegate(string name, in char[] value) sink) const { + // FIXME: print the details better + foreach(arg; invalidArguments) + sink(arg.name, arg.givenValue.toString ~ " - " ~ arg.description); + } +} + +/++ + Base class for when you've requested a feature that is not available. It may not be available because it is possible, but not yet implemented, or it might be because it is impossible on your operating system. ++/ +class FeatureUnavailableException : ArsdExceptionBase { + this(string featureName = __PRETTY_FUNCTION__, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(featureName, file, line, next); + } +} + +/++ + This means the feature could be done, but I haven't gotten around to implementing it yet. If you email me, I might be able to add it somewhat quickly and get back to you. ++/ +class NotYetImplementedException : FeatureUnavailableException { + this(string featureName = __PRETTY_FUNCTION__, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(featureName, file, line, next); + } + +} + +/++ + This means the feature is not supported by your current operating system. You might be able to get it in an update, but you might just have to find an alternate way of doing things. ++/ +class NotSupportedException : FeatureUnavailableException { + this(string featureName, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + super(featureName, file, line, next); + } +} + +/++ + This is a generic exception with attached arguments. It is used when I had to throw something but didn't want to write a new class. + + You can catch an ArsdException to get its passed arguments out. + + You can pass either a base class or a string as `Type`. + + See the examples for how to use it. ++/ +template ArsdException(alias Type, DataTuple...) { + static if(DataTuple.length) + alias Parent = ArsdException!(Type, DataTuple[0 .. $-1]); + else + alias Parent = ArsdExceptionBase; + + class ArsdException : Parent { + DataTuple data; + + this(DataTuple data, string file = __FILE__, size_t line = __LINE__) { + this.data = data; + static if(is(Parent == ArsdExceptionBase)) + super(null, file, line); + else + super(data[0 .. $-1], file, line); + } + + static opCall(R...)(R r, string file = __FILE__, size_t line = __LINE__) { + return new ArsdException!(Type, DataTuple, R)(r, file, line); + } + + override string printableExceptionName() const { + static if(DataTuple.length) + enum str = "ArsdException!(" ~ Type.stringof ~ ", " ~ DataTuple.stringof[1 .. $-1] ~ ")"; + else + enum str = "ArsdException!" ~ Type.stringof; + return str; + } + + override void getAdditionalPrintableInformation(scope void delegate(string name, in char[] value) sink) const { + ArsdExceptionBase.getAdditionalPrintableInformation(sink); + + foreach(idx, datum; data) { + enum int lol = cast(int) idx; + enum key = "[" ~ lol.stringof ~ "] " ~ DataTuple[idx].stringof; + sink(key, toStringInternal(datum)); + } + } + } +} + +/// This example shows how you can throw and catch the ad-hoc exception types. +unittest { + // you can throw and catch by matching the string and argument types + try { + // throw it with parenthesis after the template args (it uses opCall to construct) + throw ArsdException!"Test"(); + // you could also `throw new ArsdException!"test";`, but that gets harder with args + // as we'll see in the following example + assert(0); // remove from docs + } catch(ArsdException!"Test" e) { // catch it without them + // this has no useful information except for the type + // but you can catch it like this and it is still more than generic Exception + } + + // an exception's job is to deliver useful information up the chain + // and you can do that easily by passing arguments: + + try { + throw ArsdException!"Test"(4, "four"); + // you could also `throw new ArsdException!("Test", int, string)(4, "four")` + // but now you start to see how the opCall convenience constructor simplifies things + assert(0); // remove from docs + } catch(ArsdException!("Test", int, string) e) { // catch it and use info by specifying types + assert(e.data[0] == 4); // and extract arguments like this + assert(e.data[1] == "four"); + } + + // a throw site can add additional information without breaking code that catches just some + // generally speaking, each additional argument creates a new subclass on top of the previous args + // so you can cast + + try { + throw ArsdException!"Test"(4, "four", 9); + assert(0); // remove from docs + } catch(ArsdException!("Test", int, string) e) { // this catch still works + assert(e.data[0] == 4); + assert(e.data[1] == "four"); + // but if you were to print it, all the members would be there + // import std.stdio; writeln(e); // would show something like: + /+ + ArsdException!("Test", int, string, int)@file.d(line): + [0] int: 4 + [1] string: four + [2] int: 9 + +/ + // indicating that there's additional information available if you wanted to process it + + // and meanwhile: + ArsdException!("Test", int) e2 = e; // this implicit cast works thanks to the parent-child relationship + ArsdException!"Test" e3 = e; // this works too, the base type/string still matches + + // so catching those types would work too + } +} + +/++ + A tagged union that holds an error code from system apis, meaning one from Windows GetLastError() or C's errno. + + You construct it with `SystemErrorCode(thing)` and the overloaded constructor tags and stores it. ++/ +struct SystemErrorCode { + /// + enum Type { + errno, /// + win32 /// + } + + const Type type; /// + const int code; /// You should technically cast it back to DWORD if it is a win32 code + + /++ + C/unix error are typed as signed ints... + Windows' errors are typed DWORD, aka unsigned... + + so just passing them straight up will pick the right overload here to set the tag. + +/ + this(int errno) { + this.type = Type.errno; + this.code = errno; + } + + /// ditto + this(uint win32) { + this.type = Type.win32; + this.code = win32; + } + + /++ + Returns if the code indicated success. + + Please note that many calls do not actually set a code to success, but rather just don't touch it. Thus this may only be true on `init`. + +/ + bool wasSuccessful() const { + final switch(type) { + case Type.errno: + return this.code == 0; + case Type.win32: + return this.code == 0; + } + } + + /++ + Constructs a string containing both the code and the explanation string. + +/ + string toString() const { + return "[" ~ codeAsString ~ "] " ~ errorString; + } + + /++ + The numeric code itself as a string. + + See [errorString] for a text explanation of the code. + +/ + string codeAsString() const { + char[16] buffer; + final switch(type) { + case Type.errno: + return intToString(code, buffer[]).idup; + case Type.win32: + buffer[0 .. 2] = "0x"; + return buffer[0 .. 2 + intToString(cast(uint) code, buffer[2 .. $], IntToStringArgs().withRadix(16).withPadding(8)).length].idup; + } + } + + /++ + A text explanation of the code. See [codeAsString] for a string representation of the numeric representation. + +/ + string errorString() const @trusted { + final switch(type) { + case Type.errno: + import core.stdc.string; + auto strptr = strerror(code); + auto orig = strptr; + int len; + while(*strptr++) { + len++; + } + + return orig[0 .. len].idup; + case Type.win32: + version(Windows) { + wchar[256] buffer; + auto size = FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + null, + code, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buffer.ptr, + buffer.length, + null + ); + + return makeUtf8StringFromWindowsString(buffer[0 .. size]).stripInternal; + } else { + return null; + } + } + } +} + +/++ + ++/ +struct SavedArgument { + string name; + LimitedVariant value; +} + +/++ + ++/ +class SystemApiException : ArsdExceptionBase { + this(string msg, int originalErrorNo, scope SavedArgument[] args = null, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this(msg, SystemErrorCode(originalErrorNo), args, file, line, next); + } + + version(Windows) + this(string msg, DWORD windowsError, scope SavedArgument[] args = null, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this(msg, SystemErrorCode(windowsError), args, file, line, next); + } + + this(string msg, SystemErrorCode code, SavedArgument[] args = null, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.errorCode = code; + + // discard stuff that won't fit + if(args.length > this.args.length) + args = args[0 .. this.args.length]; + + this.args[0 .. args.length] = args[]; + + super(msg, file, line, next); + } + + /++ + + +/ + const SystemErrorCode errorCode; + + /++ + + +/ + const SavedArgument[8] args; + + override void getAdditionalPrintableInformation(scope void delegate(string name, in char[] value) sink) const { + super.getAdditionalPrintableInformation(sink); + sink("Error code", errorCode.toString()); + + foreach(arg; args) + if(arg.name !is null) + sink(arg.name, arg.value.toString()); + } + +} + +/++ + The low level use of this would look like `throw new WindowsApiException("MsgWaitForMultipleObjectsEx", GetLastError())` but it is meant to be used from higher level things like [Win32Enforce]. + + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +alias WindowsApiException = SystemApiException; + +/++ + History: + Moved from simpledisplay.d to core.d in March 2023 (dub v11.0). ++/ +alias ErrnoApiException = SystemApiException; + +/++ + Calls the C API function `fn`. If it returns an error value, it throws an [ErrnoApiException] (or subclass) after getting `errno`. ++/ +template ErrnoEnforce(alias fn, alias errorValue = void) { + static if(is(typeof(fn) Return == return)) + static if(is(typeof(fn) Params == __parameters)) { + static if(is(errorValue == void)) { + static if(is(typeof(null) : Return)) + enum errorValueToUse = null; + else static if(is(Return : long)) + enum errorValueToUse = -1; + else + static assert(0, "Please pass the error value"); + } else { + enum errorValueToUse = errorValue; + } + + Return ErrnoEnforce(Params params, ArgSentinel sentinel = ArgSentinel.init, string file = __FILE__, size_t line = __LINE__) { + import core.stdc.errno; + + Return value = fn(params); + + if(value == errorValueToUse) { + SavedArgument[] args; // FIXME + /+ + static foreach(idx; 0 .. Params.length) + args ~= SavedArgument( + __traits(identifier, Params[idx .. idx + 1]), + params[idx] + ); + +/ + throw new ErrnoApiException(__traits(identifier, fn), errno, args, file, line); + } + + return value; + } + } +} + +version(Windows) { + /++ + Calls the Windows API function `fn`. If it returns an error value, it throws a [WindowsApiException] (or subclass) after calling `GetLastError()`. + +/ + template Win32Enforce(alias fn, alias errorValue = void) { + static if(is(typeof(fn) Return == return)) + static if(is(typeof(fn) Params == __parameters)) { + static if(is(errorValue == void)) { + static if(is(Return == BOOL)) + enum errorValueToUse = false; + else static if(is(Return : HANDLE)) + enum errorValueToUse = NULL; + else static if(is(Return == DWORD)) + enum errorValueToUse = cast(DWORD) 0xffffffff; + else + static assert(0, "Please pass the error value"); + } else { + enum errorValueToUse = errorValue; + } + + Return Win32Enforce(Params params, ArgSentinel sentinel = ArgSentinel.init, string file = __FILE__, size_t line = __LINE__) { + Return value = fn(params); + + if(value == errorValueToUse) { + auto error = GetLastError(); + SavedArgument[] args; // FIXME + throw new WindowsApiException(__traits(identifier, fn), error, args, file, line); + } + + return value; + } + } + } + +} + +/+ + =============== + EVENT LOOP CORE + =============== ++/ + +/+ + UI threads + need to get window messages in addition to all the other jobs + I/O Worker threads + need to get commands for read/writes, run them, and send the reply back. not necessary on Windows + if interrupted, check cancel flags. + CPU Worker threads + gets functions, runs them, send reply back. should send a cancel flag to periodically check + Task worker threads + runs fibers and multiplexes them + + + General procedure: + issue the read/write command + if it would block on linux, epoll associate it. otherwise do the callback immediately + + callbacks have default affinity to the current thread, meaning their callbacks always run here + accepts can usually be dispatched to any available thread tho + + // In other words, a single thread can be associated with, at most, one I/O completion port. + + Realistically, IOCP only used if there is no thread affinity. If there is, just do overlapped w/ sleepex. + + + case study: http server + + 1) main thread starts the server. it does an accept loop with no thread affinity. the main thread does NOT check the global queue (the iocp/global epoll) + 2) connections come in and are assigned to first available thread via the iocp/global epoll + 3) these run local event loops until the connection task is finished + + EVENT LOOP TYPES: + 1) main ui thread - MsgWaitForMultipleObjectsEx / epoll on the local ui. it does NOT check the any worker thread thing! + The main ui thread should never terminate until the program is ready to close. + You can have additional ui threads in theory but im not really gonna support that in full; most things will assume there is just the one. simpledisplay's gui thread is the primary if it exists. (and sdpy will prolly continue to be threaded the way it is now) + + The biggest complication is the TerminalDirectToEmulator, where the primary ui thread is NOT the thread that runs `main` + 2) worker thread GetQueuedCompletionStatusEx / epoll on the local thread fd and the global epoll fd + 3) local event loop - check local things only. SleepEx / epoll on local thread fd. This more of a compatibility hack for `waitForCompletion` outside a fiber. + + i'll use: + * QueueUserAPC to send interruptions to a worker thread + * PostQueuedCompletionStatus is to send interruptions to any available thread. + * PostMessage to a window + * ??? to a fiber task + + I also need a way to de-duplicate events in the queue so if you try to push the same thing it won't trigger multiple times.... I might want to keep a duplicate of the thing... really, what I'd do is post the "event wake up" message and keep the queue in my own thing. (WM_PAINT auto-coalesces) + + Destructors need to be able to post messages back to a specific task to queue thread-affinity cleanup. This must be GC safe. + + A task might want to wait on certain events. If the task is a fiber, it yields and gets called upon the event. If the task is a thread, it really has to call the event loop... which can be a loop of loops we want to avoid. `waitForCompletion` is more often gonna be used just to run the loop at top level tho... it might not even check for the global info availability so it'd run the local thing only. + + APCs should not themselves enter an alterable wait cuz it can stack overflow. So generally speaking, they should avoid calling fibers or other event loops. ++/ + +/++ + You can also pass a handle to a specific thread, if you have one. ++/ +enum ThreadToRunIn { + /++ + The callback should be only run by the same thread that set it. + +/ + CurrentThread, + /++ + The UI thread is a special one - it is the supervisor of the workers and the controller of gui and console handles. It is the first thread to call [arsd_core_init] actively running an event loop unless there is a thread that has actively asserted the ui supervisor role. FIXME is this true after i implemen it? + + A ui thread should be always quickly responsive to new events. + + There should only be one main ui thread, in which simpledisplay and minigui can be used. + + Other threads can run like ui threads, but are considered temporary and only concerned with their own needs (it is the default style of loop + for an undeclared thread but will not receive messages from other threads unless there is no other option) + + + Ad-Hoc thread - something running an event loop that isn't another thing + Controller thread - running an explicit event loop instance set as not a task runner or blocking worker + UI thread - simpledisplay's event loop, which it will require remain live for the duration of the program (running two .eventLoops without a parent EventLoop instance will become illegal, throwing at runtime if it happens telling people to change their code) + + Windows HANDLES will always be listened on the thread itself that is requesting, UNLESS it is a worker/helper thread, in which case it goes to a coordinator thread. since it prolly can't rely on the parent per se this will have to be one created by arsd core init, UNLESS the parent is inside an explicit EventLoop structure. + + All use the MsgWaitForMultipleObjectsEx pattern + + + +/ + UiThread, + /++ + The callback can be called from any available worker thread. It will be added to a global queue and the first thread to see it will run it. + + These will not run on the UI thread unless there is no other option on the platform (and all platforms this lib supports have other options). + + These are expected to run cooperatively multitasked things; functions that frequently yield as they wait on other tasks. Think a fiber. + + A task runner should be generally responsive to new events. + +/ + AnyAvailableTaskRunnerThread, + /++ + These are expected to run longer blocking, but independent operations. Think an individual function with no context. + + A blocking worker can wait hundreds of milliseconds between checking for new events. + +/ + AnyAvailableBlockingWorkerThread, + /++ + The callback will be duplicated across all threads known to the arsd.core event loop. + + It adds it to an immutable queue that each thread will go through... might just replace with an exit() function. + + + so to cancel all associated tasks for like a web server, it could just have the tasks atomicAdd to a counter and subtract when they are finished. Then you have a single semaphore you signal the number of times you have an active thing and wait for them to acknowledge it. + + threads should report when they start running the loop and they really should report when they terminate but that isn't reliable + + + hmmm what if: all user-created threads (the public api) count as ui threads. only ones created in here are task runners or helpers. ui threads can wait on a global event to exit. + + there's still prolly be one "the" ui thread, which does the handle listening on windows and is the one sdpy wants. + +/ + BroadcastToAllThreads, +} + +/++ + Initializes the arsd core event loop and creates its worker threads. You don't actually have to call this, since the first use of an arsd.core function that requires it will call it implicitly, but calling it yourself gives you a chance to control the configuration more explicitly if you want to. ++/ +void arsd_core_init(int numberOfWorkers = 0) { + +} + +version(Windows) +class WindowsHandleReader_ex { + // Windows handles are always dispatched to the main ui thread, which can then send a command back to a worker thread to run the callback if needed + this(HANDLE handle) {} +} + +version(Posix) +class PosixFdReader_ex { + // posix readers can just register with whatever instance we want to handle the callback +} + +/++ + ++/ +interface ICoreEventLoop { + /++ + Runs the event loop for this thread until the `until` delegate returns `true`. + +/ + final void run(scope bool delegate() until) { + while(!exitApplicationRequested && !until()) { + runOnce(); + } + } + + private __gshared bool exitApplicationRequested; + + final static void exitApplication() { + exitApplicationRequested = true; + // FIXME: wake up all the threads + } + + /++ + Returns details from a call to [runOnce]. Use the named methods here for details, or it can be used in a `while` loop directly thanks to its `opCast` automatic conversion to `bool`. + + History: + Added December 28, 2023 + +/ + static struct RunOnceResult { + enum Possibilities { + CarryOn, + LocalExit, + GlobalExit, + Interrupted + + } + Possibilities result; + + /++ + Returns `true` if the event loop should generally continue. + + Might be false if the local loop was exited or if the application is supposed to exit. If this is `false`, check [applicationExitRequested] to determine if you should move on to other work or start your final cleanup process. + +/ + bool shouldContinue() const { + return result == Possibilities.CarryOn; + } + + /++ + Returns `true` if [ICoreEventLoop.exitApplication] was called during this event, or if the user or operating system has requested the application exit. + + Details might be available through other means. + +/ + bool applicationExitRequested() const { + return result == Possibilities.GlobalExit; + } + + /++ + Returns [shouldContinue] when used in a context for an implicit bool (e.g. `if` statements). + +/ + bool opCast(T : bool)() const { + reutrn shouldContinue(); + } + } + + /++ + Runs a single iteration of the event loop for this thread. It will return when the first thing happens, but that thing might be totally uninteresting to anyone, or it might trigger significant work you'll wait on. + + Note that running this externally instead of `run` gives only the $(I illusion) of control. You're actually better off setting a recurring timer if you need things to run on a clock tick, or a single-shot timer for a one time event. They're more likely to be called on schedule inside this function than outside it. + + Parameters: + timeout = a timeout value for an idle loop. There is no guarantee you won't return earlier or later than this; the function might run longer than the timeout if it has work to do. Pass `Duration.max` (the default) for an infinite duration timeout (but remember, once it finds work to do, including a false-positive wakeup or interruption by the operating system, it will return early anyway). + + History: + Prior to December 28, 2023, it returned `void` and took no arguments. This change is breaking, but since the entire module is documented as unstable, it was permitted to happen as that document provided prior notice. + +/ + RunOnceResult runOnce(Duration timeout = Duration.max); + + /++ + Adds a delegate to be called on each loop iteration, called based on the `timingFlags`. + + + The order in which the delegates are called is undefined and may change with each iteration of the loop. Additionally, when and how many times a loop iterates is undefined; multiple events might be handled by each iteration, or sometimes, nothing will be handled and it woke up spuriously. Your delegates need to be ok with all of this. + + Parameters: + dg = the delegate to call + timingFlags = + 0: never actually run the function; it can assert error if you pass this + 1: run before each loop OS wait call + 2: run after each loop OS wait call + 3: run both before and after each OS wait call + 4: single shot? + 8: no-coalesce? (if after was just run, it will skip the before loops unless this flag is set) + + +/ + void addDelegateOnLoopIteration(void delegate() dg, uint timingFlags); + + final void addDelegateOnLoopIteration(void function() dg, uint timingFlags) { + addDelegateOnLoopIteration(toDelegate(dg), timingFlags); + } + + // to send messages between threads, i'll queue up a function that just call dispatchMessage. can embed the arg inside the callback helper prolly. + // tho i might prefer to actually do messages w/ run payloads so it is easier to deduplicate i can still dedupe by insepcting the call args so idk + + version(Posix) { + @mustuse + static struct UnregisterToken { + private CoreEventLoopImplementation impl; + private int fd; + private CallbackHelper cb; + + /++ + Unregisters the file descriptor from the event loop and releases the reference to the callback held by the event loop (which will probably free it). + + You must call this when you're done. Normally, this will be right before you close the fd (Which is often after the other side closes it, meaning you got a 0 length read). + +/ + void unregister() { + assert(impl !is null, "Cannot reuse unregister token"); + + version(Arsd_core_epoll) { + impl.unregisterFd(fd); + } else version(Arsd_core_kqueue) { + // intentionally blank - all registrations are one-shot there + // FIXME: actually it might not have gone off yet, in that case we do need to delete the filter + } else version(EmptyCoreEvent) { + + } + else static assert(0); + + cb.release(); + this = typeof(this).init; + } + } + + @mustuse + static struct RearmToken { + private bool readable; + private CoreEventLoopImplementation impl; + private int fd; + private CallbackHelper cb; + private uint flags; + + /++ + Calls [UnregisterToken.unregister] + +/ + void unregister() { + assert(impl !is null, "cannot reuse rearm token after unregistering it"); + + version(Arsd_core_epoll) { + impl.unregisterFd(fd); + } else version(Arsd_core_kqueue) { + // intentionally blank - all registrations are one-shot there + // FIXME: actually it might not have gone off yet, in that case we do need to delete the filter + } else version(EmptyCoreEvent) { + + } else static assert(0); + + cb.release(); + this = typeof(this).init; + } + + /++ + Rearms the event so you will get another callback next time it is ready. + +/ + void rearm() { + assert(impl !is null, "cannot reuse rearm token after unregistering it"); + impl.rearmFd(this); + } + } + + UnregisterToken addCallbackOnFdReadable(int fd, CallbackHelper cb); + RearmToken addCallbackOnFdReadableOneShot(int fd, CallbackHelper cb); + RearmToken addCallbackOnFdWritableOneShot(int fd, CallbackHelper cb); + } + + version(Windows) { + @mustuse + static struct UnregisterToken { + private CoreEventLoopImplementation impl; + private HANDLE handle; + private CallbackHelper cb; + + /++ + Unregisters the handle from the event loop and releases the reference to the callback held by the event loop (which will probably free it). + + You must call this when you're done. Normally, this will be right before you close the handle. + +/ + void unregister() { + assert(impl !is null, "Cannot reuse unregister token"); + + impl.unregisterHandle(handle, cb); + + cb.release(); + this = typeof(this).init; + } + } + + UnregisterToken addCallbackOnHandleReady(HANDLE handle, CallbackHelper cb); + } +} + +/++ + Get the event loop associated with this thread ++/ +ICoreEventLoop getThisThreadEventLoop(EventLoopType type = EventLoopType.AdHoc) { + static ICoreEventLoop loop; + if(loop is null) + loop = new CoreEventLoopImplementation(); + return loop; +} + +/++ + The internal types that will be exposed through other api things. ++/ +package(arsd) enum EventLoopType { + /++ + The event loop is being run temporarily and the thread doesn't promise to keep running it. + +/ + AdHoc, + /++ + The event loop struct has been instantiated at top level. Its destructor will run when the + function exits, which is only at the end of the entire block of work it is responsible for. + + It must be in scope for the whole time the arsd event loop functions are expected to be used + (meaning it should generally be top-level in `main`) + +/ + Explicit, + /++ + A specialization of `Explicit`, so all the same rules apply there, but this is specifically the event loop coming from simpledisplay or minigui. It will run for the duration of the UI's existence. + +/ + Ui, + /++ + A special event loop specifically for threads that listen to the task runner queue and handle I/O events from running tasks. Typically, a task runner runs cooperatively multitasked coroutines (so they prefer not to block the whole thread). + +/ + TaskRunner, + /++ + A special event loop specifically for threads that listen to the helper function request queue. Helper functions are expected to run independently for a somewhat long time (them blocking the thread for some time is normal) and send a reply message back to the requester. + +/ + HelperWorker +} + +/+ + Tasks are given an object to talk to their parent... can be a dialog where it is like + + sendBuffer + waitForWordToProceed + + in a loop + + + Tasks are assigned to a worker thread and may share it with other tasks. ++/ + + +// the GC may not be able to see this! remember, it can be hidden inside kernel buffers +version(HasThread) package(arsd) class CallbackHelper { + import core.memory; + + void call() { + if(callback) + callback(); + } + + void delegate() callback; + void*[3] argsStore; + + void addref() { + atomicOp!"+="(refcount, 1); + } + + void release() { + if(atomicOp!"-="(refcount, 1) <= 0) { + if(flags & 1) + GC.removeRoot(cast(void*) this); + } + } + + private shared(int) refcount; + private uint flags; + + this(void function() callback) { + this( () { callback(); } ); + } + + this(void delegate() callback, bool addRoot = true) { + if(addRoot) { + GC.addRoot(cast(void*) this); + this.flags |= 1; + } + + this.addref(); + this.callback = callback; + } +} + +/++ + This represents a file. Technically, file paths aren't actually strings (for example, on Linux, they need not be valid utf-8, while a D string is supposed to be), even though we almost always use them like that. + + This type is meant to represent a filename / path. I might not keep it around. ++/ +struct FilePath { + string path; + + bool isNull() { + return path is null; + } + + bool opCast(T:bool)() { + return !isNull; + } + + string toString() { + return path; + } + + //alias toString this; +} + +/++ + Represents a generic async, waitable request. ++/ +class AsyncOperationRequest { + /++ + Actually issues the request, starting the operation. + +/ + abstract void start(); + /++ + Cancels the request. This will cause `isComplete` to return true once the cancellation has been processed, but [AsyncOperationResponse.wasSuccessful] will return `false` (unless it completed before the cancellation was processed, in which case it is still allowed to finish successfully). + + After cancelling a request, you should still wait for it to complete to ensure that the task has actually released its resources before doing anything else on it. + + Once a cancellation request has been sent, it cannot be undone. + +/ + abstract void cancel(); + + /++ + Returns `true` if the operation has been completed. It may be completed successfully, cancelled, or have errored out - to check this, call [waitForCompletion] and check the members on the response object. + +/ + abstract bool isComplete(); + /++ + Waits until the request has completed - successfully or otherwise - and returns the response object. It will run an ad-hoc event loop that may call other callbacks while waiting. + + The response object may be embedded in the request object - do not reuse the request until you are finished with the response and do not keep the response around longer than you keep the request. + + + Note to implementers: all subclasses should override this and return their specific response object. You can use the top-level `waitForFirstToCompleteByIndex` function with a single-element static array to help with the implementation. + +/ + abstract AsyncOperationResponse waitForCompletion(); + + /++ + + +/ + // abstract void repeat(); +} + +/++ + ++/ +interface AsyncOperationResponse { + /++ + Returns true if the request completed successfully, finishing what it was supposed to. + + Should be set to `false` if the request was cancelled before completing or encountered an error. + +/ + bool wasSuccessful(); +} + +/++ + It returns the $(I request) so you can identify it more easily. `request.waitForCompletion()` is guaranteed to return the response without any actual wait, since it is already complete when this function returns. + + Please note that "completion" is not necessary successful completion; a request being cancelled or encountering an error also counts as it being completed. + + The `waitForFirstToCompleteByIndex` version instead returns the index of the array entry that completed first. + + It is your responsibility to remove the completed request from the array before calling the function again, since any request already completed will always be immediately returned. + + You might prefer using [asTheyComplete], which will give each request as it completes and loop over until all of them are complete. + + Returns: + `null` or `requests.length` if none completed before returning. ++/ +AsyncOperationRequest waitForFirstToComplete(AsyncOperationRequest[] requests...) { + auto idx = waitForFirstToCompleteByIndex(requests); + if(idx == requests.length) + return null; + return requests[idx]; +} +/// ditto +size_t waitForFirstToCompleteByIndex(AsyncOperationRequest[] requests...) { + size_t helper() { + foreach(idx, request; requests) + if(request.isComplete()) + return idx; + return requests.length; + } + + auto idx = helper(); + // if one is already done, return it + if(idx != requests.length) + return idx; + + // otherwise, run the ad-hoc event loop until one is + // FIXME: what if we are inside a fiber? + auto el = getThisThreadEventLoop(); + el.run(() => (idx = helper()) != requests.length); + + return idx; +} + +/++ + Waits for all the `requests` to complete, giving each one through the range interface as it completes. + + This meant to be used in a foreach loop. + + The `requests` array and its contents must remain valid for the lifetime of the returned range. Its contents may be shuffled as the requests complete (the implementation works through an unstable sort+remove). ++/ +AsTheyCompleteRange asTheyComplete(AsyncOperationRequest[] requests...) { + return AsTheyCompleteRange(requests); +} +/// ditto +struct AsTheyCompleteRange { + AsyncOperationRequest[] requests; + + this(AsyncOperationRequest[] requests) { + this.requests = requests; + + if(requests.length == 0) + return; + + // wait for first one to complete, then move it to the front of the array + moveFirstCompleteToFront(); + } + + private void moveFirstCompleteToFront() { + auto idx = waitForFirstToCompleteByIndex(requests); + + auto tmp = requests[0]; + requests[0] = requests[idx]; + requests[idx] = tmp; + } + + bool empty() { + return requests.length == 0; + } + + void popFront() { + assert(!empty); + /+ + this needs to + 1) remove the front of the array as being already processed (unless it is the initial priming call) + 2) wait for one of them to complete + 3) move the complete one to the front of the array + +/ + + requests[0] = requests[$-1]; + requests = requests[0 .. $-1]; + + if(requests.length) + moveFirstCompleteToFront(); + } + + AsyncOperationRequest front() { + return requests[0]; + } +} + +version(Windows) { + alias NativeFileHandle = HANDLE; /// + alias NativeSocketHandle = SOCKET; /// + alias NativePipeHandle = HANDLE; /// +} else version(Posix) { + alias NativeFileHandle = int; /// + alias NativeSocketHandle = int; /// + alias NativePipeHandle = int; /// +} + +/++ + An `AbstractFile` represents a file handle on the operating system level. You cannot do much with it. ++/ +version(HasFile) class AbstractFile { + private { + NativeFileHandle handle; + } + + /++ + +/ + enum OpenMode { + readOnly, /// C's "r", the file is read + writeWithTruncation, /// C's "w", the file is blanked upon opening so it only holds what you write + appendOnly, /// C's "a", writes will always be appended to the file + readAndWrite /// C's "r+", writes will overwrite existing parts of the file based on where you seek (default is at the beginning) + } + + /++ + +/ + enum RequirePreexisting { + no, + yes + } + + /+ + enum SpecialFlags { + randomAccessExpected, /// FILE_FLAG_SEQUENTIAL_SCAN is turned off and posix_fadvise(POSIX_FADV_SEQUENTIAL) + skipCache, /// O_DSYNC, FILE_FLAG_NO_BUFFERING and maybe WRITE_THROUGH. note that metadata still goes through the cache, FlushFileBuffers and fsync can still do those + temporary, /// FILE_ATTRIBUTE_TEMPORARY on Windows, idk how to specify on linux. also FILE_FLAG_DELETE_ON_CLOSE can be combined to make a (almost) all memory file. kinda like a private anonymous mmap i believe. + deleteWhenClosed, /// Windows has a flag for this but idk if it is of any real use + async, /// open it in overlapped mode, all reads and writes must then provide an offset. Only implemented on Windows + } + +/ + + /++ + + +/ + protected this(bool async, FilePath filename, OpenMode mode = OpenMode.readOnly, RequirePreexisting require = RequirePreexisting.no, uint specialFlags = 0) { + version(Windows) { + DWORD access; + DWORD creation; + + final switch(mode) { + case OpenMode.readOnly: + access = GENERIC_READ; + creation = OPEN_EXISTING; + break; + case OpenMode.writeWithTruncation: + access = GENERIC_WRITE; + + final switch(require) { + case RequirePreexisting.no: + creation = CREATE_ALWAYS; + break; + case RequirePreexisting.yes: + creation = TRUNCATE_EXISTING; + break; + } + break; + case OpenMode.appendOnly: + access = FILE_APPEND_DATA; + + final switch(require) { + case RequirePreexisting.no: + creation = CREATE_ALWAYS; + break; + case RequirePreexisting.yes: + creation = OPEN_EXISTING; + break; + } + break; + case OpenMode.readAndWrite: + access = GENERIC_READ | GENERIC_WRITE; + + final switch(require) { + case RequirePreexisting.no: + creation = CREATE_NEW; + break; + case RequirePreexisting.yes: + creation = OPEN_EXISTING; + break; + } + break; + } + + WCharzBuffer wname = WCharzBuffer(filename.path); + + auto handle = CreateFileW( + wname.ptr, + access, + FILE_SHARE_READ, + null, + creation, + FILE_ATTRIBUTE_NORMAL | (async ? FILE_FLAG_OVERLAPPED : 0), + null + ); + + if(handle == INVALID_HANDLE_VALUE) { + // FIXME: throw the filename and other params here too + SavedArgument[3] args; + args[0] = SavedArgument("filename", LimitedVariant(filename.path)); + args[1] = SavedArgument("access", LimitedVariant(access, 2)); + args[2] = SavedArgument("requirePreexisting", LimitedVariant(require == RequirePreexisting.yes)); + throw new WindowsApiException("CreateFileW", GetLastError(), args[]); + } + + this.handle = handle; + } else version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + + CharzBuffer namez = CharzBuffer(filename.path); + int flags; + + // FIXME does mac not have cloexec for real or is this just a druntime problem????? + version(Arsd_core_has_cloexec) { + flags = O_CLOEXEC; + } else { + scope(success) + setCloExec(this.handle); + } + + if(async) + flags |= O_NONBLOCK; + + final switch(mode) { + case OpenMode.readOnly: + flags |= O_RDONLY; + break; + case OpenMode.writeWithTruncation: + flags |= O_WRONLY | O_TRUNC; + + final switch(require) { + case RequirePreexisting.no: + flags |= O_CREAT; + break; + case RequirePreexisting.yes: + break; + } + break; + case OpenMode.appendOnly: + flags |= O_APPEND; + + final switch(require) { + case RequirePreexisting.no: + flags |= O_CREAT; + break; + case RequirePreexisting.yes: + break; + } + break; + case OpenMode.readAndWrite: + flags |= O_RDWR; + + final switch(require) { + case RequirePreexisting.no: + flags |= O_CREAT; + break; + case RequirePreexisting.yes: + break; + } + break; + } + + auto perms = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + int fd = open(namez.ptr, flags, perms); + if(fd == -1) { + SavedArgument[3] args; + args[0] = SavedArgument("filename", LimitedVariant(filename.path)); + args[1] = SavedArgument("flags", LimitedVariant(flags, 2)); + args[2] = SavedArgument("perms", LimitedVariant(perms, 8)); + throw new ErrnoApiException("open", errno, args[]); + } + + this.handle = fd; + } + } + + /++ + + +/ + private this(NativeFileHandle handleToWrap) { + this.handle = handleToWrap; + } + + // only available on some types of file + long size() { return 0; } + + // note that there is no fsync thing, instead use the special flag. + + /++ + + +/ + void close() { + version(Windows) { + Win32Enforce!CloseHandle(handle); + handle = null; + } else version(Posix) { + import unix = core.sys.posix.unistd; + import core.sys.posix.fcntl; + + ErrnoEnforce!(unix.close)(handle); + handle = -1; + } + } +} + +/++ + ++/ +version(HasFile) class File : AbstractFile { + + /++ + Opens a file in synchronous access mode. + + The permission mask is on used on posix systems FIXME: implement it + +/ + this(FilePath filename, OpenMode mode = OpenMode.readOnly, RequirePreexisting require = RequirePreexisting.no, uint specialFlags = 0, uint permMask = 0) { + super(false, filename, mode, require, specialFlags); + } + + /++ + + +/ + ubyte[] read(scope ubyte[] buffer) { + return null; + } + + /++ + + +/ + void write(in void[] buffer) { + } + + enum Seek { + current, + fromBeginning, + fromEnd + } + + // Seeking/telling/sizing is not permitted when appending and some files don't support it + // also not permitted in async mode + void seek(long where, Seek fromWhence) {} + long tell() { return 0; } +} + +/++ + Only one operation can be pending at any time in the current implementation. ++/ +version(HasFile) class AsyncFile : AbstractFile { + /++ + Opens a file in asynchronous access mode. + +/ + this(FilePath filename, OpenMode mode = OpenMode.readOnly, RequirePreexisting require = RequirePreexisting.no, uint specialFlags = 0, uint permissionMask = 0) { + // FIXME: implement permissionMask + super(true, filename, mode, require, specialFlags); + } + + package(arsd) this(NativeFileHandle adoptPreSetup) { + super(adoptPreSetup); + } + + /// + AsyncReadRequest read(ubyte[] buffer, long offset = 0) { + return new AsyncReadRequest(this, buffer, offset); + } + + /// + AsyncWriteRequest write(const(void)[] buffer, long offset = 0) { + return new AsyncWriteRequest(this, cast(ubyte[]) buffer, offset); + } + +} + +/++ + Reads or writes a file in one call. It might internally yield, but is generally blocking if it returns values. The callback ones depend on the implementation. + + Tip: prefer the callback ones. If settings where async is possible, it will do async, and if not, it will sync. + + NOT IMPLEMENTED ++/ +void writeFile(string filename, const(void)[] contents) { + +} + +/// ditto +string readTextFile(string filename, string fileEncoding = null) { + return null; +} + +/// ditto +const(ubyte[]) readBinaryFile(string filename) { + return null; +} + +/+ +private Class recycleObject(Class, Args...)(Class objectToRecycle, Args args) { + if(objectToRecycle is null) + return new Class(args); + // destroy nulls out the vtable which is the first thing in the object + // so if it hasn't already been destroyed, we'll do it here + if((*cast(void**) objectToRecycle) !is null) { + assert(typeid(objectToRecycle) is typeid(Class)); // to make sure we're actually recycling the right kind of object + .destroy(objectToRecycle); + } + + // then go ahead and reinitialize it + ubyte[] rawData = (cast(ubyte*) cast(void*) objectToRecycle)[0 .. __traits(classInstanceSize, Class)]; + rawData[] = (cast(ubyte[]) typeid(Class).initializer)[]; + + objectToRecycle.__ctor(args); + + return objectToRecycle; +} ++/ + +/+ +/++ + Preallocates a class object without initializing it. + + This is suitable *only* for passing to one of the functions in here that takes a preallocated object for recycling. ++/ +Class preallocate(Class)() { + import core.memory; + // FIXME: can i pass NO_SCAN here? + return cast(Class) GC.calloc(__traits(classInstanceSize, Class), 0, typeid(Class)); +} + +OwnedClass!Class preallocateOnStack(Class)() { + +} ++/ + +// thanks for a random person on stack overflow for this function +version(Windows) +BOOL MyCreatePipeEx( + PHANDLE lpReadPipe, + PHANDLE lpWritePipe, + LPSECURITY_ATTRIBUTES lpPipeAttributes, + DWORD nSize, + DWORD dwReadMode, + DWORD dwWriteMode +) +{ + HANDLE ReadPipeHandle, WritePipeHandle; + DWORD dwError; + CHAR[MAX_PATH] PipeNameBuffer; + + if (nSize == 0) { + nSize = 4096; + } + + // FIXME: should be atomic op and gshared + static shared(int) PipeSerialNumber = 0; + + import core.stdc.string; + import core.stdc.stdio; + + sprintf(PipeNameBuffer.ptr, + "\\\\.\\Pipe\\ArsdCoreAnonymousPipe.%08x.%08x".ptr, + GetCurrentProcessId(), + atomicOp!"+="(PipeSerialNumber, 1) + ); + + ReadPipeHandle = CreateNamedPipeA( + PipeNameBuffer.ptr, + 1/*PIPE_ACCESS_INBOUND*/ | dwReadMode, + 0/*PIPE_TYPE_BYTE*/ | 0/*PIPE_WAIT*/, + 1, // Number of pipes + nSize, // Out buffer size + nSize, // In buffer size + 120 * 1000, // Timeout in ms + lpPipeAttributes + ); + + if (! ReadPipeHandle) { + return FALSE; + } + + WritePipeHandle = CreateFileA( + PipeNameBuffer.ptr, + GENERIC_WRITE, + 0, // No sharing + lpPipeAttributes, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | dwWriteMode, + null // Template file + ); + + if (INVALID_HANDLE_VALUE == WritePipeHandle) { + dwError = GetLastError(); + CloseHandle( ReadPipeHandle ); + SetLastError(dwError); + return FALSE; + } + + *lpReadPipe = ReadPipeHandle; + *lpWritePipe = WritePipeHandle; + return( TRUE ); +} + + + +/+ + + // this is probably useless. + +/++ + Creates a pair of anonymous pipes ready for async operations. + + You can pass some preallocated objects to recycle if you like. ++/ +AsyncAnonymousPipe[2] anonymousPipePair(AsyncAnonymousPipe[2] preallocatedObjects = [null, null], bool inheritable = false) { + version(Posix) { + int[2] fds; + auto ret = pipe(fds); + + if(ret == -1) + throw new SystemApiException("pipe", errno); + + // FIXME: do we want them inheritable? and do we want both sides to be async? + if(!inheritable) { + setCloExec(fds[0]); + setCloExec(fds[1]); + } + // if it is inherited, do we actually want it non-blocking? + makeNonBlocking(fds[0]); + makeNonBlocking(fds[1]); + + return [ + recycleObject(preallocatedObjects[0], fds[0]), + recycleObject(preallocatedObjects[1], fds[1]), + ]; + } else version(Windows) { + HANDLE rp, wp; + // FIXME: do we want them inheritable? and do we want both sides to be async? + if(!MyCreatePipeEx(&rp, &wp, null, 0, FILE_FLAG_OVERLAPPED, FILE_FLAG_OVERLAPPED)) + throw new SystemApiException("MyCreatePipeEx", GetLastError()); + return [ + recycleObject(preallocatedObjects[0], rp), + recycleObject(preallocatedObjects[1], wp), + ]; + } else throw ArsdException!"NotYetImplemented"(); +} + // on posix, just do pipe() w/ non block + // on windows, do an overlapped named pipe server, connect, stop listening, return pair. ++/ + +/+ +class NamedPipe : AsyncFile { + +} ++/ + +/++ + A named pipe ready to accept connections. + + A Windows named pipe is an IPC mechanism usable on local machines or across a Windows network. ++/ +version(Windows) +class NamedPipeServer { + // unix domain socket or windows named pipe + + // Promise!AsyncAnonymousPipe connect; + // Promise!AsyncAnonymousPipe accept; + + // when a new connection arrives, it calls your callback + // can be on a specific thread or on any thread +} + +private version(Windows) extern(Windows) { + const(char)* inet_ntop(int, const void*, char*, socklen_t); +} + +/++ + Some functions that return arrays allow you to provide your own buffer. These are indicated in the type system as `UserProvidedBuffer!Type`, and you get to decide what you want to happen if the buffer is too small via the [OnOutOfSpace] parameter. + + These are usually optional, since an empty user provided buffer with the default policy of reallocate will also work fine for whatever needs to be returned, thanks to the garbage collector taking care of it for you. + + The API inside `UserProvidedBuffer` is all private to the arsd library implementation; your job is just to provide the buffer to it with [provideBuffer] or a constructor call and decide on your on-out-of-space policy. + + $(TIP + To properly size a buffer, I suggest looking at what covers about 80% of cases. Trying to cover everything often leads to wasted buffer space, and if you use a reallocate policy it can cover the rest. You might be surprised how far just two elements can go! + ) + + History: + Added August 4, 2023 (dub v11.0) ++/ +struct UserProvidedBuffer(T) { + private T[] buffer; + private int actualLength; + private OnOutOfSpace policy; + + /++ + + +/ + public this(scope T[] buffer, OnOutOfSpace policy = OnOutOfSpace.reallocate) { + this.buffer = buffer; + this.policy = policy; + } + + package(arsd) bool append(T item) { + if(actualLength < buffer.length) { + buffer[actualLength++] = item; + return true; + } else final switch(policy) { + case OnOutOfSpace.discard: + return false; + case OnOutOfSpace.exception: + throw ArsdException!"Buffer out of space"(buffer.length, actualLength); + case OnOutOfSpace.reallocate: + buffer ~= item; + actualLength++; + return true; + } + } + + package(arsd) T[] slice() return { + return buffer[0 .. actualLength]; + } +} + +/// ditto +UserProvidedBuffer!T provideBuffer(T)(scope T[] buffer, OnOutOfSpace policy = OnOutOfSpace.reallocate) { + return UserProvidedBuffer!T(buffer, policy); +} + +/++ + Possible policies for [UserProvidedBuffer]s that run out of space. ++/ +enum OnOutOfSpace { + reallocate, /// reallocate the buffer with the GC to make room + discard, /// discard all contents that do not fit in your provided buffer + exception, /// throw an exception if there is data that would not fit in your provided buffer +} + + + +/+ + The GC can be called from any thread, and a lot of cleanup must be done + on the gui thread. Since the GC can interrupt any locks - including being + triggered inside a critical section - it is vital to avoid deadlocks to get + these functions called from the right place. + + If the buffer overflows, things are going to get leaked. I'm kinda ok with that + right now. + + The cleanup function is run when the event loop gets around to it, which is just + whenever there's something there after it has been woken up for other work. It does + NOT wake up the loop itself - can't risk doing that from inside the GC in another thread. + (Well actually it might be ok but i don't wanna mess with it right now.) ++/ +package(arsd) struct CleanupQueue { + import core.stdc.stdlib; + + void queue(alias func, T...)(T args) { + static struct Args { + T args; + } + static struct RealJob { + Job j; + Args a; + } + static void call(Job* data) { + auto rj = cast(RealJob*) data; + func(rj.a.args); + } + + RealJob* thing = cast(RealJob*) malloc(RealJob.sizeof); + thing.j.call = &call; + thing.a.args = args; + + buffer[tail++] = cast(Job*) thing; + + // FIXME: set overflowed + } + + void process() { + const tail = this.tail; + + while(tail != head) { + Job* job = cast(Job*) buffer[head++]; + job.call(job); + free(job); + } + + if(overflowed) + throw new object.Exception("cleanup overflowed"); + } + + private: + + ubyte tail; // must ONLY be written by queue + ubyte head; // must ONLY be written by process + bool overflowed; + + static struct Job { + void function(Job*) call; + } + + void*[256] buffer; +} +package(arsd) __gshared CleanupQueue cleanupQueue; + + + + +/++ + A timer that will trigger your function on a given interval. + + + You create a timer with an interval and a callback. It will continue + to fire on the interval until it is destroyed. + + --- + auto timer = new Timer(50, { it happened!; }); + timer.destroy(); + --- + + Timers can only be expected to fire when the event loop is running and only + once per iteration through the event loop. + + History: + Prior to December 9, 2020, a timer pulse set too high with a handler too + slow could lock up the event loop. It now guarantees other things will + get a chance to run between timer calls, even if that means not keeping up + with the requested interval. ++/ +version(HasTimer) +class Timer { + // FIXME: absolute time vs relative time + // FIXME: real time? + + // FIXME: I might add overloads for ones that take a count of + // how many elapsed since last time (on Windows, it will divide + // the ticks thing given, on Linux it is just available) and + // maybe one that takes an instance of the Timer itself too + + + /++ + Creates an initialized, but unarmed timer. You must call other methods later. + +/ + this() { + initialize(); + } + + private void initialize() { + version(Windows) { + handle = CreateWaitableTimer(null, false, null); + if(handle is null) + throw new WindowsApiException("CreateWaitableTimer", GetLastError()); + cbh = new CallbackHelper(&trigger); + } else version(linux) { + import core.sys.linux.timerfd; + + fd = timerfd_create(CLOCK_MONOTONIC, 0); + if(fd == -1) + throw new Exception("timer create failed"); + + auto el = getThisThreadEventLoop(EventLoopType.Ui); + unregisterToken = el.addCallbackOnFdReadable(fd, new CallbackHelper(&trigger)); + } else throw new NotYetImplementedException(); + // FIXME: freebsd 12 has timer_fd and netbsd 10 too + } + + /++ + +/ + void setPulseCallback(void delegate() onPulse) { + assert(onPulse !is null); + this.onPulse = onPulse; + } + + /++ + +/ + void changeTime(int intervalInMilliseconds, bool repeats) { + this.intervalInMilliseconds = intervalInMilliseconds; + this.repeats = repeats; + changeTimeInternal(intervalInMilliseconds, repeats); + } + + private void changeTimeInternal(int intervalInMilliseconds, bool repeats) { + version(Windows) + { + LARGE_INTEGER initialTime; + initialTime.QuadPart = -intervalInMilliseconds * 10000000L / 1000; // Windows wants hnsecs, we have msecs + if(!SetWaitableTimer(handle, &initialTime, repeats ? intervalInMilliseconds : 0, &timerCallback, cast(void*) cbh, false)) + throw new WindowsApiException("SetWaitableTimer", GetLastError()); + } else version(linux) { + import core.sys.linux.timerfd; + + itimerspec value = makeItimerspec(intervalInMilliseconds, repeats); + if(timerfd_settime(fd, 0, &value, null) == -1) { + throw new ErrnoApiException("couldn't change pulse timer", errno); + } + } else { + throw new NotYetImplementedException(); + } + // FIXME: freebsd 12 has timer_fd and netbsd 10 too + } + + /++ + +/ + void pause() { + // FIXME this kinda makes little sense tbh + // when it restarts, it won't be on the same rhythm as it was at first... + changeTimeInternal(0, false); + } + + /++ + +/ + void unpause() { + changeTimeInternal(this.intervalInMilliseconds, this.repeats); + } + + /++ + +/ + void cancel() { + version(Windows) + CancelWaitableTimer(handle); + else + changeTime(0, false); + } + + + /++ + Create a timer with a callback when it triggers. + +/ + this(int intervalInMilliseconds, void delegate() onPulse, bool repeats = true) @trusted { + assert(onPulse !is null); + + initialize(); + setPulseCallback(onPulse); + changeTime(intervalInMilliseconds, repeats); + } + + version(Windows) {} else { + ICoreEventLoop.UnregisterToken unregisterToken; + } + + // just cuz I sometimes call it this. + alias dispose = destroy; + + /++ + Stop and destroy the timer object. + + You should not use it again after destroying it. + +/ + void destroy() { + version(Windows) { + cbh.release(); + } else { + unregisterToken.unregister(); + } + + version(Windows) { + staticDestroy(handle); + handle = null; + } else version(linux) { + staticDestroy(fd); + fd = -1; + } else throw new NotYetImplementedException(); + } + + ~this() { + version(Windows) {} else + cleanupQueue.queue!unregister(unregisterToken); + version(Windows) { if(handle) + cleanupQueue.queue!staticDestroy(handle); + } else version(linux) { if(fd != -1) + cleanupQueue.queue!staticDestroy(fd); + } + } + + + private: + + version(Windows) + static void staticDestroy(HANDLE handle) { + if(handle) { + // KillTimer(null, handle); + CancelWaitableTimer(cast(void*)handle); + CloseHandle(handle); + } + } + else version(linux) + static void staticDestroy(int fd) @system { + if(fd != -1) { + import unix = core.sys.posix.unistd; + + unix.close(fd); + } + } + + version(Windows) {} else + static void unregister(arsd.core.ICoreEventLoop.UnregisterToken urt) { + urt.unregister(); + } + + + void delegate() onPulse; + int intervalInMilliseconds; + bool repeats; + + int lastEventLoopRoundTriggered; + + version(linux) { + static auto makeItimerspec(int intervalInMilliseconds, bool repeats) { + import core.sys.linux.timerfd; + + itimerspec value; + value.it_value.tv_sec = cast(int) (intervalInMilliseconds / 1000); + value.it_value.tv_nsec = (intervalInMilliseconds % 1000) * 1000_000; + + if(repeats) { + value.it_interval.tv_sec = cast(int) (intervalInMilliseconds / 1000); + value.it_interval.tv_nsec = (intervalInMilliseconds % 1000) * 1000_000; + } + + return value; + } + } + + void trigger() { + version(linux) { + import unix = core.sys.posix.unistd; + long val; + unix.read(fd, &val, val.sizeof); // gotta clear the pipe + } else version(Windows) { + if(this.lastEventLoopRoundTriggered == eventLoopRound) + return; // never try to actually run faster than the event loop + lastEventLoopRoundTriggered = eventLoopRound; + } else throw new NotYetImplementedException(); + + if(onPulse) + onPulse(); + } + + version(Windows) + extern(Windows) + //static void timerCallback(HWND, UINT, UINT_PTR timer, DWORD dwTime) nothrow { + static void timerCallback(void* timer, DWORD lowTime, DWORD hiTime) nothrow { + auto cbh = cast(CallbackHelper) timer; + try + cbh.call(); + catch(Throwable e) { sdpy_abort(e); assert(0); } + } + + version(Windows) { + HANDLE handle; + CallbackHelper cbh; + } else version(linux) { + int fd = -1; + } else version(OSXCocoa) { + } else static assert(0, "timer not supported"); +} + +version(Windows) + private void sdpy_abort(Throwable e) nothrow { + try + MessageBoxA(null, (e.toString() ~ "\0").ptr, "Exception caught in WndProc", 0); + catch(Exception e) + MessageBoxA(null, "Exception.toString threw too!", "Exception caught in WndProc", 0); + ExitProcess(1); + } + + +private int eventLoopRound = -1; // so things that assume 0 still work eg lastEventLoopRoundTriggered + + + +/++ + For functions that give you an unknown address, you can use this to hold it. + + Can get: + ip4 + ip6 + unix + abstract_ + + name lookup for connect (stream or dgram) + request canonical name? + + interface lookup for bind (stream or dgram) ++/ +version(HasSocket) struct SocketAddress { + import core.sys.posix.netdb; + + /++ + Provides the set of addresses to listen on all supported protocols on the machine for the given interfaces. `localhost` only listens on the loopback interface, whereas `allInterfaces` will listen on loopback as well as the others on the system (meaning it may be publicly exposed to the internet). + + If you provide a buffer, I recommend using one of length two, so `SocketAddress[2]`, since this usually provides one address for ipv4 and one for ipv6. + +/ + static SocketAddress[] localhost(ushort port, return UserProvidedBuffer!SocketAddress buffer = null) { + buffer.append(ip6("::1", port)); + buffer.append(ip4("127.0.0.1", port)); + return buffer.slice; + } + + /// ditto + static SocketAddress[] allInterfaces(ushort port, return UserProvidedBuffer!SocketAddress buffer = null) { + char[16] str; + return allInterfaces(intToString(port, str[]), buffer); + } + + /// ditto + static SocketAddress[] allInterfaces(scope const char[] serviceOrPort, return UserProvidedBuffer!SocketAddress buffer = null) { + addrinfo hints; + hints.ai_flags = AI_PASSIVE; + hints.ai_socktype = SOCK_STREAM; // just to filter it down a little tbh + return get(null, serviceOrPort, &hints, buffer); + } + + /++ + Returns a single address object for the given protocol and parameters. + + You probably should generally prefer [get], [localhost], or [allInterfaces] to have more flexible code. + +/ + static SocketAddress ip4(scope const char[] address, ushort port, bool forListening = false) { + return getSingleAddress(AF_INET, AI_NUMERICHOST | (forListening ? AI_PASSIVE : 0), address, port); + } + + /// ditto + static SocketAddress ip4(ushort port) { + return ip4(null, port, true); + } + + /// ditto + static SocketAddress ip6(scope const char[] address, ushort port, bool forListening = false) { + return getSingleAddress(AF_INET6, AI_NUMERICHOST | (forListening ? AI_PASSIVE : 0), address, port); + } + + /// ditto + static SocketAddress ip6(ushort port) { + return ip6(null, port, true); + } + + /// ditto + static SocketAddress unix(scope const char[] path) { + // FIXME + SocketAddress addr; + return addr; + } + + /// ditto + static SocketAddress abstract_(scope const char[] path) { + char[190] buffer = void; + buffer[0] = 0; + buffer[1 .. path.length] = path[]; + return unix(buffer[0 .. 1 + path.length]); + } + + private static SocketAddress getSingleAddress(int family, int flags, scope const char[] address, ushort port) { + addrinfo hints; + hints.ai_family = family; + hints.ai_flags = flags; + + char[16] portBuffer; + char[] portString = intToString(port, portBuffer[]); + + SocketAddress[1] addr; + auto res = get(address, portString, &hints, provideBuffer(addr[])); + if(res.length == 0) + throw ArsdException!"bad address"(address.idup, port); + return res[0]; + } + + /++ + Calls `getaddrinfo` and returns the array of results. It will populate the data into the buffer you provide, if you provide one, otherwise it will allocate its own. + +/ + static SocketAddress[] get(scope const char[] nodeName, scope const char[] serviceOrPort, addrinfo* hints = null, return UserProvidedBuffer!SocketAddress buffer = null, scope bool delegate(scope addrinfo* ai) filter = null) @trusted { + addrinfo* res; + CharzBuffer node = nodeName; + CharzBuffer service = serviceOrPort; + auto ret = getaddrinfo(nodeName is null ? null : node.ptr, serviceOrPort is null ? null : service.ptr, hints, &res); + if(ret == 0) { + auto current = res; + while(current) { + if(filter is null || filter(current)) { + SocketAddress addr; + addr.addrlen = cast(socklen_t) current.ai_addrlen; + switch(current.ai_family) { + case AF_INET: + addr.in4 = * cast(sockaddr_in*) current.ai_addr; + break; + case AF_INET6: + addr.in6 = * cast(sockaddr_in6*) current.ai_addr; + break; + case AF_UNIX: + addr.unix_address = * cast(sockaddr_un*) current.ai_addr; + break; + default: + // skip + } + + if(!buffer.append(addr)) + break; + } + + current = current.ai_next; + } + + freeaddrinfo(res); + } else { + version(Windows) { + throw new WindowsApiException("getaddrinfo", ret); + } else { + const char* error = gai_strerror(ret); + } + } + + return buffer.slice; + } + + /++ + Returns a string representation of the address that identifies it in a custom format. + + $(LIST + * Unix domain socket addresses are their path prefixed with "unix:", unless they are in the abstract namespace, in which case it is prefixed with "abstract:" and the zero is trimmed out. For example, "unix:/tmp/pipe". + + * IPv4 addresses are written in dotted decimal followed by a colon and the port number. For example, "127.0.0.1:8080". + + * IPv6 addresses are written in colon separated hex format, but enclosed in brackets, then followed by the colon and port number. For example, "[::1]:8080". + ) + +/ + string toString() const @trusted { + char[200] buffer; + switch(address.sa_family) { + case AF_INET: + auto writable = stringz(inet_ntop(address.sa_family, &in4.sin_addr, buffer.ptr, buffer.length)); + auto it = writable.borrow; + buffer[it.length] = ':'; + auto numbers = intToString(port, buffer[it.length + 1 .. $]); + return buffer[0 .. it.length + 1 + numbers.length].idup; + case AF_INET6: + buffer[0] = '['; + auto writable = stringz(inet_ntop(address.sa_family, &in6.sin6_addr, buffer.ptr + 1, buffer.length - 1)); + auto it = writable.borrow; + buffer[it.length + 1] = ']'; + buffer[it.length + 2] = ':'; + auto numbers = intToString(port, buffer[it.length + 3 .. $]); + return buffer[0 .. it.length + 3 + numbers.length].idup; + case AF_UNIX: + // FIXME: it might be abstract in which case stringz is wrong!!!!! + auto writable = stringz(cast(char*) unix_address.sun_path.ptr).borrow; + if(writable.length == 0) + return "unix:"; + string prefix = writable[0] == 0 ? "abstract:" : "unix:"; + buffer[0 .. prefix.length] = prefix[]; + buffer[prefix.length .. prefix.length + writable.length] = writable[writable[0] == 0 ? 1 : 0 .. $]; + return buffer.idup; + case AF_UNSPEC: + return ""; + default: + return ""; // FIXME + } + } + + ushort port() const @trusted { + switch(address.sa_family) { + case AF_INET: + return ntohs(in4.sin_port); + case AF_INET6: + return ntohs(in6.sin6_port); + default: + return 0; + } + } + + /+ + @safe unittest { + SocketAddress[4] buffer; + foreach(addr; SocketAddress.get("arsdnet.net", "http", null, provideBuffer(buffer[]))) + writeln(addr.toString()); + } + +/ + + /+ + unittest { + // writeln(SocketAddress.ip4(null, 4444, true)); + // writeln(SocketAddress.ip4("400.3.2.1", 4444)); + // writeln(SocketAddress.ip4("bar", 4444)); + foreach(addr; localhost(4444)) + writeln(addr.toString()); + } + +/ + + socklen_t addrlen = typeof(this).sizeof - socklen_t.sizeof; // the size of the union below + + union { + sockaddr address; + + sockaddr_storage storage; + + sockaddr_in in4; + sockaddr_in6 in6; + + sockaddr_un unix_address; + } + + /+ + this(string node, string serviceOrPort, int family = 0) { + // need to populate the approrpiate address and the length and make sure you set sa_family + } + +/ + + int domain() { + return address.sa_family; + } + sockaddr* rawAddr() return { + return &address; + } + socklen_t rawAddrLength() { + return addrlen; + } + + // FIXME it is AF_BLUETOOTH + // see: https://people.csail.mit.edu/albert/bluez-intro/x79.html + // see: https://learn.microsoft.com/en-us/windows/win32/Bluetooth/bluetooth-programming-with-windows-sockets +} + +private version(Windows) { + struct sockaddr_un { + ushort sun_family; + char[108] sun_path; + } +} + +version(HasFile) class AsyncSocket : AsyncFile { + // otherwise: accept, bind, connect, shutdown, close. + + static auto lastError() { + version(Windows) + return WSAGetLastError(); + else + return errno; + } + + static bool wouldHaveBlocked() { + auto error = lastError; + version(Windows) { + return error == WSAEWOULDBLOCK || error == WSAETIMEDOUT; + } else { + return error == EAGAIN || error == EWOULDBLOCK; + } + } + + version(Windows) + enum INVALID = INVALID_SOCKET; + else + enum INVALID = -1; + + // type is mostly SOCK_STREAM or SOCK_DGRAM + /++ + Creates a socket compatible with the given address. It does not actually connect or bind, nor store the address. You will want to pass it again to those functions: + + --- + auto socket = new Socket(address, Socket.Type.Stream); + socket.connect(address).waitForCompletion(); + --- + +/ + this(SocketAddress address, int type, int protocol = 0) { + // need to look up these values for linux + // type |= SOCK_NONBLOCK | SOCK_CLOEXEC; + + handle_ = socket(address.domain(), type, protocol); + if(handle == INVALID) + throw new SystemApiException("socket", lastError()); + + super(cast(NativeFileHandle) handle); // I think that cast is ok on Windows... i think + + version(Posix) { + makeNonBlocking(handle); + setCloExec(handle); + } + + if(address.domain == AF_INET6) { + int opt = 1; + setsockopt(handle, IPPROTO_IPV6 /*SOL_IPV6*/, IPV6_V6ONLY, &opt, opt.sizeof); + } + + // FIXME: chekc for broadcast + + // FIXME: REUSEADDR ? + + // FIXME: also set NO_DELAY prolly + // int opt = 1; + // setsockopt(handle, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } + + /++ + Enabling NODELAY can give latency improvements if you are managing buffers on your end + +/ + void setNoDelay(bool enabled) { + + } + + /++ + + `allowQuickRestart` will set the SO_REUSEADDR on unix and SO_DONTLINGER on Windows, + allowing the application to be quickly restarted despite there still potentially being + pending data in the tcp stack. + + See https://stackoverflow.com/questions/3229860/what-is-the-meaning-of-so-reuseaddr-setsockopt-option-linux for more information. + + If you already set your appropriate socket options or value correctness and reliability of the network stream over restart speed, leave this at the default `false`. + +/ + void bind(SocketAddress address, bool allowQuickRestart = false) { + if(allowQuickRestart) { + // FIXME + } + + auto ret = .bind(handle, address.rawAddr, address.rawAddrLength); + if(ret == -1) + throw new SystemApiException("bind", lastError); + } + + /++ + You must call [bind] before this. + + The backlog should be set to a value where your application can reliably catch up on the backlog in a reasonable amount of time under average load. It is meant to smooth over short duration bursts and making it too big will leave clients hanging - which might cause them to try to reconnect, thinking things got lost in transit, adding to your impossible backlog. + + I personally tend to set this to be two per worker thread unless I have actual real world measurements saying to do something else. It is a bit arbitrary and not based on legitimate reasoning, it just seems to work for me (perhaps just because it has never really been put to the test). + +/ + void listen(int backlog) { + auto ret = .listen(handle, backlog); + if(ret == -1) + throw new SystemApiException("listen", lastError); + } + + /++ + +/ + void shutdown(int how) { + auto ret = .shutdown(handle, how); + if(ret == -1) + throw new SystemApiException("shutdown", lastError); + } + + /++ + +/ + override void close() { + version(Windows) + closesocket(handle); + else + .close(handle); + handle_ = -1; + } + + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncConnectRequest connect(SocketAddress address, ubyte[] bufferToSend = null) { + return new AsyncConnectRequest(this, address, bufferToSend); + } + + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncAcceptRequest accept() { + return new AsyncAcceptRequest(this); + } + + // note that send is just sendto w/ a null address + // and receive is just receivefrom w/ a null address + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncSendRequest send(const(ubyte)[] buffer, int flags = 0) { + return new AsyncSendRequest(this, buffer, null, flags); + } + + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncReceiveRequest receive(ubyte[] buffer, int flags = 0) { + return new AsyncReceiveRequest(this, buffer, null, flags); + } + + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncSendRequest sendTo(const(ubyte)[] buffer, SocketAddress* address, int flags = 0) { + return new AsyncSendRequest(this, buffer, address, flags); + } + /++ + You can also construct your own request externally to control the memory more. + +/ + AsyncReceiveRequest receiveFrom(ubyte[] buffer, SocketAddress* address, int flags = 0) { + return new AsyncReceiveRequest(this, buffer, address, flags); + } + + /++ + +/ + SocketAddress localAddress() { + SocketAddress addr; + getsockname(handle, &addr.address, &addr.addrlen); + return addr; + } + /++ + +/ + SocketAddress peerAddress() { + SocketAddress addr; + getpeername(handle, &addr.address, &addr.addrlen); + return addr; + } + + // for unix sockets on unix only: send/receive fd, get peer creds + + /++ + + +/ + final NativeSocketHandle handle() { + return handle_; + } + + private NativeSocketHandle handle_; +} + +/++ + Initiates a connection request and optionally sends initial data as soon as possible. + + Calls `ConnectEx` on Windows and emulates it on other systems. + + The entire buffer is sent before the operation is considered complete. + + NOT IMPLEMENTED / NOT STABLE ++/ +version(HasSocket) class AsyncConnectRequest : AsyncOperationRequest { + // FIXME: i should take a list of addresses and take the first one that succeeds, so a getaddrinfo can be sent straight in. + this(AsyncSocket socket, SocketAddress address, ubyte[] dataToWrite) { + + } + + override void start() {} + override void cancel() {} + override bool isComplete() { return true; } + override AsyncConnectResponse waitForCompletion() { assert(0); } +} +/++ ++/ +version(HasSocket) class AsyncConnectResponse : AsyncOperationResponse { + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode) { + this.errorCode = errorCode; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } + +} + +// FIXME: TransmitFile/sendfile support + +/++ + Calls `AcceptEx` on Windows and emulates it on other systems. + + NOT IMPLEMENTED / NOT STABLE ++/ +version(HasSocket) class AsyncAcceptRequest : AsyncOperationRequest { + AsyncSocket socket; + + override void start() {} + override void cancel() {} + override bool isComplete() { return true; } + override AsyncConnectResponse waitForCompletion() { assert(0); } + + + struct LowLevelOperation { + AsyncSocket file; + ubyte[] buffer; + SocketAddress* address; + + this(typeof(this.tupleof) args) { + this.tupleof = args; + } + + version(Windows) { + auto opCall(OVERLAPPED* overlapped, LPOVERLAPPED_COMPLETION_ROUTINE ocr) { + WSABUF buf; + buf.len = cast(int) buffer.length; + buf.buf = cast(typeof(buf.buf)) buffer.ptr; + + uint flags; + + if(address is null) + return WSARecv(file.handle, &buf, 1, null, &flags, overlapped, ocr); + else { + return WSARecvFrom(file.handle, &buf, 1, null, &flags, &(address.address), &(address.addrlen), overlapped, ocr); + } + } + } else { + auto opCall() { + int flags; + if(address is null) + return core.sys.posix.sys.socket.recv(file.handle, buffer.ptr, buffer.length, flags); + else + return core.sys.posix.sys.socket.recvfrom(file.handle, buffer.ptr, buffer.length, flags, &(address.address), &(address.addrlen)); + } + } + + string errorString() { + return "Receive"; + } + } + mixin OverlappedIoRequest!(AsyncAcceptResponse, LowLevelOperation); + + this(AsyncSocket socket, ubyte[] buffer = null, SocketAddress* address = null) { + llo = LowLevelOperation(socket, buffer, address); + this.response = typeof(this.response).defaultConstructed; + } + + // can also look up the local address +} +/++ ++/ +version(HasSocket) class AsyncAcceptResponse : AsyncOperationResponse { + AsyncSocket newSocket; + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode, ubyte[] buffer) { + this.errorCode = errorCode; + } + + this(AsyncSocket newSocket, SystemErrorCode errorCode) { + this.newSocket = newSocket; + this.errorCode = errorCode; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } +} + +/++ ++/ +version(HasSocket) class AsyncReceiveRequest : AsyncOperationRequest { + struct LowLevelOperation { + AsyncSocket file; + ubyte[] buffer; + int flags; + SocketAddress* address; + + this(typeof(this.tupleof) args) { + this.tupleof = args; + } + + version(Windows) { + auto opCall(OVERLAPPED* overlapped, LPOVERLAPPED_COMPLETION_ROUTINE ocr) { + WSABUF buf; + buf.len = cast(int) buffer.length; + buf.buf = cast(typeof(buf.buf)) buffer.ptr; + + uint flags = this.flags; + + if(address is null) + return WSARecv(file.handle, &buf, 1, null, &flags, overlapped, ocr); + else { + return WSARecvFrom(file.handle, &buf, 1, null, &flags, &(address.address), &(address.addrlen), overlapped, ocr); + } + } + } else { + auto opCall() { + if(address is null) + return core.sys.posix.sys.socket.recv(file.handle, buffer.ptr, buffer.length, flags); + else + return core.sys.posix.sys.socket.recvfrom(file.handle, buffer.ptr, buffer.length, flags, &(address.address), &(address.addrlen)); + } + } + + string errorString() { + return "Receive"; + } + } + mixin OverlappedIoRequest!(AsyncReceiveResponse, LowLevelOperation); + + this(AsyncSocket socket, ubyte[] buffer, SocketAddress* address, int flags) { + llo = LowLevelOperation(socket, buffer, flags, address); + this.response = typeof(this.response).defaultConstructed; + } + +} +/++ ++/ +version(HasSocket) class AsyncReceiveResponse : AsyncOperationResponse { + const ubyte[] bufferWritten; + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode, const(ubyte)[] bufferWritten) { + this.errorCode = errorCode; + this.bufferWritten = bufferWritten; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } +} + +/++ ++/ +version(HasSocket) class AsyncSendRequest : AsyncOperationRequest { + struct LowLevelOperation { + AsyncSocket file; + const(ubyte)[] buffer; + int flags; + SocketAddress* address; + + this(typeof(this.tupleof) args) { + this.tupleof = args; + } + + version(Windows) { + auto opCall(OVERLAPPED* overlapped, LPOVERLAPPED_COMPLETION_ROUTINE ocr) { + WSABUF buf; + buf.len = cast(int) buffer.length; + buf.buf = cast(typeof(buf.buf)) buffer.ptr; + + if(address is null) + return WSASend(file.handle, &buf, 1, null, flags, overlapped, ocr); + else { + return WSASendTo(file.handle, &buf, 1, null, flags, address.rawAddr, address.rawAddrLength, overlapped, ocr); + } + } + } else { + auto opCall() { + if(address is null) + return core.sys.posix.sys.socket.send(file.handle, buffer.ptr, buffer.length, flags); + else + return core.sys.posix.sys.socket.sendto(file.handle, buffer.ptr, buffer.length, flags, address.rawAddr, address.rawAddrLength); + } + } + + string errorString() { + return "Send"; + } + } + mixin OverlappedIoRequest!(AsyncSendResponse, LowLevelOperation); + + this(AsyncSocket socket, const(ubyte)[] buffer, SocketAddress* address, int flags) { + llo = LowLevelOperation(socket, buffer, flags, address); + this.response = typeof(this.response).defaultConstructed; + } +} + +/++ ++/ +version(HasSocket) class AsyncSendResponse : AsyncOperationResponse { + const ubyte[] bufferWritten; + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode, const(ubyte)[] bufferWritten) { + this.errorCode = errorCode; + this.bufferWritten = bufferWritten; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } + +} + +/++ + A set of sockets bound and ready to accept connections on worker threads. + + Depending on the specified address, it can be tcp, tcpv6, unix domain, or all of the above. + + NOT IMPLEMENTED / NOT STABLE ++/ +version(HasSocket) class StreamServer { + AsyncSocket[] sockets; + + this(SocketAddress[] listenTo, int backlog = 8) { + foreach(listen; listenTo) { + auto socket = new AsyncSocket(listen, SOCK_STREAM); + + // FIXME: allInterfaces for ipv6 also covers ipv4 so the bind can fail... + // so we have to permit it to fail w/ address in use if we know we already + // are listening to ipv6 + + // or there is a setsockopt ipv6 only thing i could set. + + socket.bind(listen); + socket.listen(backlog); + sockets ~= socket; + + // writeln(socket.localAddress.port); + } + + // i have to start accepting on each thread for each socket... + } + // when a new connection arrives, it calls your callback + // can be on a specific thread or on any thread + + + void start() { + foreach(socket; sockets) { + auto request = socket.accept(); + request.start(); + } + } +} + +/+ +unittest { + auto ss = new StreamServer(SocketAddress.localhost(0)); +} ++/ + +/++ + A socket bound and ready to use receiveFrom + + Depending on the address, it can be udp or unix domain. + + NOT IMPLEMENTED / NOT STABLE ++/ +version(HasSocket) class DatagramListener { + // whenever a udp message arrives, it calls your callback + // can be on a specific thread or on any thread + + // UDP is realistically just an async read on the bound socket + // just it can get the "from" data out and might need the "more in packet" flag +} + +/++ + Just in case I decide to change the implementation some day. ++/ +version(HasFile) alias AsyncAnonymousPipe = AsyncFile; + + +// AsyncAnonymousPipe connectNamedPipe(AsyncAnonymousPipe preallocated, string name) + +// unix fifos are considered just non-seekable files and have no special support in the lib; open them as a regular file w/ the async flag. + +// DIRECTORY LISTINGS + // not async, so if you want that, do it in a helper thread + // just a convenient function to have (tho phobos has a decent one too, importing it expensive af) + +/++ + Note that the order of items called for your delegate is undefined; if you want it sorted, you'll have to collect and sort yourself. But it *might* be sorted by the OS (on Windows, it almost always is), so consider that when choosing a sorting algorithm. + + History: + previously in minigui as a private function. Moved to arsd.core on April 3, 2023 ++/ +version(HasFile) GetFilesResult getFiles(string directory, scope void delegate(string name, bool isDirectory) dg) { + // FIXME: my buffers here aren't great lol + + SavedArgument[1] argsForException() { + return [ + SavedArgument("directory", LimitedVariant(directory)), + ]; + } + + version(Windows) { + WIN32_FIND_DATA data; + // FIXME: if directory ends with / or \\ ? + WCharzBuffer search = WCharzBuffer(directory ~ "/*"); + auto handle = FindFirstFileW(search.ptr, &data); + scope(exit) if(handle !is INVALID_HANDLE_VALUE) FindClose(handle); + if(handle is INVALID_HANDLE_VALUE) { + if(GetLastError() == ERROR_FILE_NOT_FOUND) + return GetFilesResult.fileNotFound; + throw new WindowsApiException("FindFirstFileW", GetLastError(), argsForException()[]); + } + + try_more: + + string name = makeUtf8StringFromWindowsString(data.cFileName[0 .. findIndexOfZero(data.cFileName[])]); + + dg(name, (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? true : false); + + auto ret = FindNextFileW(handle, &data); + if(ret == 0) { + if(GetLastError() == ERROR_NO_MORE_FILES) + return GetFilesResult.success; + throw new WindowsApiException("FindNextFileW", GetLastError(), argsForException()[]); + } + + goto try_more; + + } else version(Posix) { + import core.sys.posix.dirent; + import core.stdc.errno; + auto dir = opendir((directory ~ "\0").ptr); + scope(exit) + if(dir) closedir(dir); + if(dir is null) + throw new ErrnoApiException("opendir", errno, argsForException()); + + auto dirent = readdir(dir); + if(dirent is null) + return GetFilesResult.fileNotFound; + + try_more: + + string name = dirent.d_name[0 .. findIndexOfZero(dirent.d_name[])].idup; + + dg(name, dirent.d_type == DT_DIR); + + dirent = readdir(dir); + if(dirent is null) + return GetFilesResult.success; + + goto try_more; + } else static assert(0); +} + +/// ditto +enum GetFilesResult { + success, + fileNotFound +} + +/++ + This is currently a simplified glob where only the * wildcard in the first or last position gets special treatment or a single * in the middle. + + More things may be added later to be more like what Phobos supports. ++/ +bool matchesFilePattern(scope const(char)[] name, scope const(char)[] pattern) { + if(pattern.length == 0) + return false; + if(pattern == "*") + return true; + if(pattern.length > 2 && pattern[0] == '*' && pattern[$-1] == '*') { + // if the rest of pattern appears in name, it is good + return name.indexOf(pattern[1 .. $-1]) != -1; + } else if(pattern[0] == '*') { + // if the rest of pattern is at end of name, it is good + return name.endsWith(pattern[1 .. $]); + } else if(pattern[$-1] == '*') { + // if the rest of pattern is at start of name, it is good + return name.startsWith(pattern[0 .. $-1]); + } else if(pattern.length >= 3) { + auto idx = pattern.indexOf("*"); + if(idx != -1) { + auto lhs = pattern[0 .. idx]; + auto rhs = pattern[idx + 1 .. $]; + if(name.length >= lhs.length + rhs.length) { + return name.startsWith(lhs) && name.endsWith(rhs); + } else { + return false; + } + } + } + + return name == pattern; +} + +unittest { + assert("test.html".matchesFilePattern("*")); + assert("test.html".matchesFilePattern("*.html")); + assert("test.html".matchesFilePattern("*.*")); + assert("test.html".matchesFilePattern("test.*")); + assert(!"test.html".matchesFilePattern("pest.*")); + assert(!"test.html".matchesFilePattern("*.dhtml")); + + assert("test.html".matchesFilePattern("t*.html")); + assert(!"test.html".matchesFilePattern("e*.html")); +} + +package(arsd) int indexOf(scope const(char)[] haystack, scope const(char)[] needle) { + if(haystack.length < needle.length) + return -1; + if(haystack == needle) + return 0; + foreach(i; 0 .. haystack.length - needle.length + 1) + if(haystack[i .. i + needle.length] == needle) + return cast(int) i; + return -1; +} + +package(arsd) int indexOf(scope const(ubyte)[] haystack, scope const(char)[] needle) { + return indexOf(cast(const(char)[]) haystack, needle); +} + +unittest { + assert("foo".indexOf("f") == 0); + assert("foo".indexOf("o") == 1); + assert("foo".indexOf("foo") == 0); + assert("foo".indexOf("oo") == 1); + assert("foo".indexOf("fo") == 0); + assert("foo".indexOf("boo") == -1); + assert("foo".indexOf("food") == -1); +} + +package(arsd) bool endsWith(scope const(char)[] haystack, scope const(char)[] needle) { + if(needle.length > haystack.length) + return false; + return haystack[$ - needle.length .. $] == needle; +} + +unittest { + assert("foo".endsWith("o")); + assert("foo".endsWith("oo")); + assert("foo".endsWith("foo")); + assert(!"foo".endsWith("food")); + assert(!"foo".endsWith("d")); +} + +package(arsd) bool startsWith(scope const(char)[] haystack, scope const(char)[] needle) { + if(needle.length > haystack.length) + return false; + return haystack[0 .. needle.length] == needle; +} + +unittest { + assert("foo".startsWith("f")); + assert("foo".startsWith("fo")); + assert("foo".startsWith("foo")); + assert(!"foo".startsWith("food")); + assert(!"foo".startsWith("d")); +} + + +// FILE/DIR WATCHES + // linux does it by name, windows and bsd do it by handle/descriptor + // dispatches change event to either your thread or maybe the any task` queue. + +/++ + PARTIALLY IMPLEMENTED / NOT STABLE + ++/ +class DirectoryWatcher { + private { + version(Arsd_core_windows) { + OVERLAPPED overlapped; + HANDLE hDirectory; + ubyte[] buffer; + + extern(Windows) + static void overlappedCompletionRoutine(DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED lpOverlapped) { + typeof(this) rr = cast(typeof(this)) (cast(void*) lpOverlapped - typeof(this).overlapped.offsetof); + + // dwErrorCode + auto response = rr.buffer[0 .. dwNumberOfBytesTransferred]; + + while(response.length) { + auto fni = cast(FILE_NOTIFY_INFORMATION*) response.ptr; + auto filename = fni.FileName[0 .. fni.FileNameLength]; + + if(fni.NextEntryOffset) + response = response[fni.NextEntryOffset .. $]; + else + response = response[$..$]; + + // FIXME: I think I need to pin every overlapped op while it is pending + // and unpin it when it is returned. GC.addRoot... but i don't wanna do that + // every op so i guess i should do a refcount scheme similar to the other callback helper. + + rr.changeHandler( + FilePath(makeUtf8StringFromWindowsString(filename)), // FIXME: this is a relative path + ChangeOperation.unknown // FIXME this is fni.Action + ); + } + + rr.requestRead(); + } + + void requestRead() { + DWORD ignored; + if(!ReadDirectoryChangesW( + hDirectory, + buffer.ptr, + cast(int) buffer.length, + recursive, + FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_CREATION | FILE_NOTIFY_CHANGE_FILE_NAME, + &ignored, + &overlapped, + &overlappedCompletionRoutine + )) { + auto error = GetLastError(); + /+ + if(error == ERROR_IO_PENDING) { + // not expected here, the docs say it returns true when queued + } + +/ + + throw new SystemApiException("ReadDirectoryChangesW", error); + } + } + } else version(Arsd_core_epoll) { + static int inotifyfd = -1; // this is TLS since it is associated with the thread's event loop + static ICoreEventLoop.UnregisterToken inotifyToken; + static CallbackHelper inotifycb; + static DirectoryWatcher[int] watchMappings; + + static ~this() { + if(inotifyfd != -1) { + close(inotifyfd); + inotifyfd = -1; + } + } + + import core.sys.linux.sys.inotify; + + int watchId = -1; + + static void inotifyReady() { + // read from it + ubyte[256 /* NAME_MAX + 1 */ + inotify_event.sizeof] sbuffer; + + auto ret = read(inotifyfd, sbuffer.ptr, sbuffer.length); + if(ret == -1) { + auto errno = errno; + if(errno == EAGAIN || errno == EWOULDBLOCK) + return; + throw new SystemApiException("read inotify", errno); + } else if(ret == 0) { + assert(0, "I don't think this is ever supposed to happen"); + } + + auto buffer = sbuffer[0 .. ret]; + + while(buffer.length > 0) { + inotify_event* event = cast(inotify_event*) buffer.ptr; + buffer = buffer[inotify_event.sizeof .. $]; + char[] filename = cast(char[]) buffer[0 .. event.len]; + buffer = buffer[event.len .. $]; + + // note that filename is padded with zeroes, so it is actually a stringz + + if(auto obj = event.wd in watchMappings) { + (*obj).changeHandler( + FilePath(stringz(filename.ptr).borrow.idup), // FIXME: this is a relative path + ChangeOperation.unknown // FIXME + ); + } else { + // it has probably already been removed + } + } + } + } else version(Arsd_core_kqueue) { + int fd; + CallbackHelper cb; + } + + FilePath path; + string globPattern; + bool recursive; + void delegate(FilePath filename, ChangeOperation op) changeHandler; + } + + enum ChangeOperation { + unknown, + deleted, // NOTE_DELETE, IN_DELETE, FILE_NOTIFY_CHANGE_FILE_NAME + written, // NOTE_WRITE / NOTE_EXTEND / NOTE_TRUNCATE, IN_MODIFY, FILE_NOTIFY_CHANGE_LAST_WRITE / FILE_NOTIFY_CHANGE_SIZE + renamed, // NOTE_RENAME, the moved from/to in linux, FILE_NOTIFY_CHANGE_FILE_NAME + metadataChanged // NOTE_ATTRIB, IN_ATTRIB, FILE_NOTIFY_CHANGE_ATTRIBUTES + + // there is a NOTE_OPEN on freebsd 13, and the access change on Windows. and an open thing on linux. so maybe i can do note open/note_read too. + } + + /+ + Windows and Linux work best when you watch directories. The operating system tells you the name of files as they change. + + BSD doesn't support this. You can only get names and reports when a file is modified by watching specific files. AS such, when you watch a directory on those systems, your delegate will be called with a null path. Cross-platform applications should check for this and not assume the name is always usable. + + inotify is kinda clearly the best of the bunch, with Windows in second place, and kqueue dead last. + + + If path to watch is a directory, it signals when a file inside the directory (only one layer deep) is created or modified. This is the most efficient on Windows and Linux. + + If a path is a file, it only signals when that specific file is written. This is most efficient on BSD. + + + The delegate is called when something happens. Note that the path modified may not be accurate on all systems when you are watching a directory. + +/ + + /++ + Watches a directory and its contents. If the `globPattern` is `null`, it will not attempt to add child items but also will not filter it, meaning you will be left with platform-specific behavior. + + On Windows, the globPattern is just used to filter events. + + On Linux, the `recursive` flag, if set, will cause it to add additional OS-level watches for each subdirectory. + + On BSD, anything other than a null pattern will cause a directory scan to add files to the watch list. + + For best results, use the most limited thing you need, as watches can get quite involved on the bsd systems. + + Newly added files and subdirectories may not be automatically added in all cases, meaning if it is added and then subsequently modified, you might miss a notification. + + If the event queue is too busy, the OS may skip a notification. + + You should always offer some way for the user to force a refresh and not rely on notifications being present; they are a convenience when they work, not an always reliable method. + +/ + this(FilePath directoryToWatch, string globPattern, bool recursive, void delegate(FilePath pathModified, ChangeOperation op) dg) { + this.path = directoryToWatch; + this.globPattern = globPattern; + this.recursive = recursive; + this.changeHandler = dg; + + version(Arsd_core_windows) { + WCharzBuffer wname = directoryToWatch.path; + buffer = new ubyte[](1024); + hDirectory = CreateFileW( + wname.ptr, + GENERIC_READ, + FILE_SHARE_READ, + null, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED | FILE_FLAG_BACKUP_SEMANTICS, + null + ); + if(hDirectory == INVALID_HANDLE_VALUE) + throw new SystemApiException("CreateFileW", GetLastError()); + + requestRead(); + } else version(Arsd_core_epoll) { + auto el = getThisThreadEventLoop(); + + // no need for sync because it is thread-local + if(inotifyfd == -1) { + inotifyfd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); + if(inotifyfd == -1) + throw new SystemApiException("inotify_init1", errno); + + inotifycb = new CallbackHelper(&inotifyReady); + inotifyToken = el.addCallbackOnFdReadable(inotifyfd, inotifycb); + } + + uint event_mask = IN_CREATE | IN_MODIFY | IN_DELETE; // FIXME + CharzBuffer dtw = directoryToWatch.path; + auto watchId = inotify_add_watch(inotifyfd, dtw.ptr, event_mask); + if(watchId < -1) + throw new SystemApiException("inotify_add_watch", errno, [SavedArgument("path", LimitedVariant(directoryToWatch.path))]); + + watchMappings[watchId] = this; + + // FIXME: recursive needs to add child things individually + + } else version(Arsd_core_kqueue) { + auto el = cast(CoreEventLoopImplementation) getThisThreadEventLoop(); + + // FIXME: need to scan for globPattern + // when a new file is added, i'll have to diff my list to detect it and open it too + // and recursive might need to scan down too. + + kevent_t ev; + + import core.sys.posix.fcntl; + CharzBuffer buffer = CharzBuffer(directoryToWatch.path); + fd = ErrnoEnforce!open(buffer.ptr, O_RDONLY); + setCloExec(fd); + + cb = new CallbackHelper(&triggered); + + EV_SET(&ev, fd, EVFILT_VNODE, EV_ADD | EV_ENABLE | EV_CLEAR, NOTE_WRITE, 0, cast(void*) cb); + ErrnoEnforce!kevent(el.kqueuefd, &ev, 1, null, 0, null); + } else assert(0, "Not yet implemented for this platform"); + } + + private void triggered() { + writeln("triggered"); + } + + void dispose() { + version(Arsd_core_windows) { + CloseHandle(hDirectory); + } else version(Arsd_core_epoll) { + watchMappings.remove(watchId); // I could also do this on the IN_IGNORE notification but idk + inotify_rm_watch(inotifyfd, watchId); + } else version(Arsd_core_kqueue) { + ErrnoEnforce!close(fd); + fd = -1; + } + } +} + +version(none) +void main() { + + // auto file = new AsyncFile(FilePath("test.txt"), AsyncFile.OpenMode.writeWithTruncation, AsyncFile.RequirePreexisting.yes); + + /+ + getFiles("c:/windows\\", (string filename, bool isDirectory) { + writeln(filename, " ", isDirectory ? "[dir]": "[file]"); + }); + +/ + + auto w = new DirectoryWatcher(FilePath("."), "*", false, (path, op) { + writeln(path.path); + }); + getThisThreadEventLoop().run(() => false); +} + +/++ + This starts up a local pipe. If it is already claimed, it just communicates with the existing one through the interface. ++/ +class SingleInstanceApplication { + // FIXME +} + +version(none) +void main() { + + auto file = new AsyncFile(FilePath("test.txt"), AsyncFile.OpenMode.writeWithTruncation, AsyncFile.RequirePreexisting.yes); + + auto buffer = cast(ubyte[]) "hello"; + auto wr = new AsyncWriteRequest(file, buffer, 0); + wr.start(); + + wr.waitForCompletion(); + + file.close(); +} + +/++ + Implementation details of some requests. You shouldn't need to know any of this, the interface is all public. ++/ +mixin template OverlappedIoRequest(Response, LowLevelOperation) { + private { + LowLevelOperation llo; + + OwnedClass!Response response; + + version(Windows) { + OVERLAPPED overlapped; + + extern(Windows) + static void overlappedCompletionRoutine(DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED lpOverlapped) { + typeof(this) rr = cast(typeof(this)) (cast(void*) lpOverlapped - typeof(this).overlapped.offsetof); + + rr.response = typeof(rr.response)(SystemErrorCode(dwErrorCode), rr.llo.buffer[0 .. dwNumberOfBytesTransferred]); + rr.state_ = State.complete; + + // FIXME: on complete? + + // this will queue our CallbackHelper and that should be run at the end of the event loop after it is woken up by the APC run + } + } + + version(Posix) { + ICoreEventLoop.RearmToken eventRegistration; + CallbackHelper cb; + + final CallbackHelper getCb() { + if(cb is null) + cb = new CallbackHelper(&cbImpl); + return cb; + } + + final void cbImpl() { + // it is ready to complete, time to do it + auto ret = llo(); + markCompleted(ret, errno); + } + + void markCompleted(long ret, int errno) { + // maybe i should queue an apc to actually do it, to ensure the event loop has cycled... FIXME + if(ret == -1) + response = typeof(response)(SystemErrorCode(errno), null); + else + response = typeof(response)(SystemErrorCode(0), llo.buffer[0 .. cast(size_t) ret]); + state_ = State.complete; + } + } + } + + enum State { + unused, + started, + inProgress, + complete + } + private State state_; + + override void start() { + assert(state_ == State.unused); + + state_ = State.started; + + version(Windows) { + if(llo(&overlapped, &overlappedCompletionRoutine)) { + // all good, though GetLastError() might have some informative info + } else { + // operation failed, the operation is always ReadFileEx or WriteFileEx so it won't give the io pending thing here + // should i issue error async? idk + state_ = State.complete; + throw new SystemApiException(llo.errorString(), GetLastError()); + } + + // ReadFileEx always queues, even if it completed synchronously. I *could* check the get overlapped result and sleepex here but i'm prolly better off just letting the event loop do its thing anyway. + } else version(Posix) { + + // first try to just do it + auto ret = llo(); + + auto errno = errno; + if(ret == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { // unable to complete right now, register and try when it is ready + eventRegistration = getThisThreadEventLoop().addCallbackOnFdReadableOneShot(this.llo.file.handle, this.getCb); + } else { + // i could set errors sync or async and since it couldn't even start, i think a sync exception is the right way + if(ret == -1) + throw new SystemApiException(llo.errorString(), errno); + markCompleted(ret, errno); // it completed synchronously (if it is an error nor not is handled by the completion handler) + } + } + } + + + override void cancel() { + if(state_ == State.complete) + return; // it has already finished, just leave it alone, no point discarding what is already done + version(Windows) { + if(state_ != State.unused) + Win32Enforce!CancelIoEx(llo.file.AbstractFile.handle, &overlapped); + // Windows will notify us when the cancellation is complete, so we need to wait for that before updating the state + } else version(Posix) { + if(state_ != State.unused) + eventRegistration.unregister(); + markCompleted(-1, ECANCELED); + } + } + + override bool isComplete() { + // just always let the event loop do it instead + return state_ == State.complete; + + /+ + version(Windows) { + return HasOverlappedIoCompleted(&overlapped); + } else version(Posix) { + return state_ == State.complete; + + } + +/ + } + + override Response waitForCompletion() { + if(state_ == State.unused) + start(); + + // FIXME: if we are inside a fiber, we can set a oncomplete callback and then yield instead... + if(state_ != State.complete) + getThisThreadEventLoop().run(&isComplete); + + /+ + version(Windows) { + SleepEx(INFINITE, true); + + //DWORD numberTransferred; + //Win32Enforce!GetOverlappedResult(file.handle, &overlapped, &numberTransferred, true); + } else version(Posix) { + getThisThreadEventLoop().run(&isComplete); + } + +/ + + return response; + } +} + +/++ + You can write to a file asynchronously by creating one of these. ++/ +version(HasSocket) final class AsyncWriteRequest : AsyncOperationRequest { + struct LowLevelOperation { + AsyncFile file; + ubyte[] buffer; + long offset; + + this(typeof(this.tupleof) args) { + this.tupleof = args; + } + + version(Windows) { + auto opCall(OVERLAPPED* overlapped, LPOVERLAPPED_COMPLETION_ROUTINE ocr) { + overlapped.Offset = (cast(ulong) offset) & 0xffff_ffff; + overlapped.OffsetHigh = ((cast(ulong) offset) >> 32) & 0xffff_ffff; + return WriteFileEx(file.handle, buffer.ptr, cast(int) buffer.length, overlapped, ocr); + } + } else { + auto opCall() { + return core.sys.posix.unistd.write(file.handle, buffer.ptr, buffer.length); + } + } + + string errorString() { + return "Write"; + } + } + mixin OverlappedIoRequest!(AsyncWriteResponse, LowLevelOperation); + + this(AsyncFile file, ubyte[] buffer, long offset) { + this.llo = LowLevelOperation(file, buffer, offset); + response = typeof(response).defaultConstructed; + } +} + +/++ + ++/ +class AsyncWriteResponse : AsyncOperationResponse { + const ubyte[] bufferWritten; + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode, const(ubyte)[] bufferWritten) { + this.errorCode = errorCode; + this.bufferWritten = bufferWritten; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } +} + +/++ + ++/ +version(HasSocket) final class AsyncReadRequest : AsyncOperationRequest { + struct LowLevelOperation { + AsyncFile file; + ubyte[] buffer; + long offset; + + this(typeof(this.tupleof) args) { + this.tupleof = args; + } + + version(Windows) { + auto opCall(OVERLAPPED* overlapped, LPOVERLAPPED_COMPLETION_ROUTINE ocr) { + overlapped.Offset = (cast(ulong) offset) & 0xffff_ffff; + overlapped.OffsetHigh = ((cast(ulong) offset) >> 32) & 0xffff_ffff; + return ReadFileEx(file.handle, buffer.ptr, cast(int) buffer.length, overlapped, ocr); + } + } else { + auto opCall() { + return core.sys.posix.unistd.read(file.handle, buffer.ptr, buffer.length); + } + } + + string errorString() { + return "Read"; + } + } + mixin OverlappedIoRequest!(AsyncReadResponse, LowLevelOperation); + + /++ + The file must have the overlapped flag enabled on Windows and the nonblock flag set on Posix. + + The buffer MUST NOT be touched by you - not used by another request, modified, read, or freed, including letting a static array going out of scope - until this request's `isComplete` returns `true`. + + The offset is where to start reading a disk file. For all other types of files, pass 0. + +/ + this(AsyncFile file, ubyte[] buffer, long offset) { + this.llo = LowLevelOperation(file, buffer, offset); + response = typeof(response).defaultConstructed; + } + + /++ + + +/ + // abstract void repeat(); +} + +/++ + ++/ +class AsyncReadResponse : AsyncOperationResponse { + const ubyte[] bufferRead; + const SystemErrorCode errorCode; + + this(SystemErrorCode errorCode, const(ubyte)[] bufferRead) { + this.errorCode = errorCode; + this.bufferRead = bufferRead; + } + + override bool wasSuccessful() { + return errorCode.wasSuccessful; + } +} + +/+ + Tasks: + startTask() + startSubTask() - what if it just did this when it knows it is being run from inside a task? + runHelperFunction() - whomever it reports to is the parent ++/ + +version(HasThread) class SchedulableTask : Fiber { + private void delegate() dg; + + // linked list stuff + private static SchedulableTask taskRoot; + private SchedulableTask previous; + private SchedulableTask next; + + // need the controlling thread to know how to wake it up if it receives a message + private Thread controllingThread; + + // the api + + this(void delegate() dg) { + assert(dg !is null); + + this.dg = dg; + super(&taskRunner); + + if(taskRoot !is null) { + this.next = taskRoot; + taskRoot.previous = this; + } + taskRoot = this; + } + + /+ + enum BehaviorOnCtrlC { + ignore, + cancel, + deliverMessage + } + +/ + + private bool cancelled; + + public void cancel() { + this.cancelled = true; + // if this is running, we can throw immediately + // otherwise if we're calling from an appropriate thread, we can call it immediately + // otherwise we need to queue a wakeup to its own thread. + // tbh we should prolly just queue it every time + } + + private void taskRunner() { + try { + dg(); + } catch(TaskCancelledException tce) { + // this space intentionally left blank; + // the purpose of this exception is to just + // let the fiber's destructors run before we + // let it die. + } catch(Throwable t) { + if(taskUncaughtException is null) { + throw t; + } else { + taskUncaughtException(t); + } + } finally { + if(this is taskRoot) { + taskRoot = taskRoot.next; + if(taskRoot !is null) + taskRoot.previous = null; + } else { + assert(this.previous !is null); + assert(this.previous.next is this); + this.previous.next = this.next; + if(this.next !is null) + this.next.previous = this.previous; + } + } + } +} + +/++ + ++/ +void delegate(Throwable t) taskUncaughtException; + +/++ + Gets an object that lets you control a schedulable task (which is a specialization of a fiber) and can be used in an `if` statement. + + --- + if(auto controller = inSchedulableTask()) { + controller.yieldUntilReadable(...); + } + --- + + History: + Added August 11, 2023 (dub v11.1) ++/ +version(HasThread) SchedulableTaskController inSchedulableTask() { + import core.thread.fiber; + + if(auto fiber = Fiber.getThis) { + return SchedulableTaskController(cast(SchedulableTask) fiber); + } + + return SchedulableTaskController(null); +} + +/// ditto +version(HasThread) struct SchedulableTaskController { + private this(SchedulableTask fiber) { + this.fiber = fiber; + } + + private SchedulableTask fiber; + + /++ + + +/ + bool opCast(T : bool)() { + return fiber !is null; + } + + /++ + + +/ + version(Posix) + void yieldUntilReadable(NativeFileHandle handle) { + assert(fiber !is null); + + auto cb = new CallbackHelper(() { fiber.call(); }); + + // FIXME: if the fd is already registered in this thread it can throw... + version(Windows) + auto rearmToken = getThisThreadEventLoop().addCallbackOnFdReadableOneShot(handle, cb); + else + auto rearmToken = getThisThreadEventLoop().addCallbackOnFdReadableOneShot(handle, cb); + + // FIXME: this is only valid if the fiber is only ever going to run in this thread! + fiber.yield(); + + rearmToken.unregister(); + + // what if there are other messages, like a ctrl+c? + if(fiber.cancelled) + throw new TaskCancelledException(); + } + + version(Windows) + void yieldUntilSignaled(NativeFileHandle handle) { + // add it to the WaitForMultipleObjects thing w/ a cb + } +} + +class TaskCancelledException : object.Exception { + this() { + super("Task cancelled"); + } +} + +version(HasThread) private class CoreWorkerThread : Thread { + this(EventLoopType type) { + this.type = type; + + // task runners are supposed to have smallish stacks since they either just run a single callback or call into fibers + // the helper runners might be a bit bigger tho + super(&run); + } + void run() { + eventLoop = getThisThreadEventLoop(this.type); + atomicOp!"+="(startedCount, 1); + atomicOp!"+="(runningCount, 1); + scope(exit) { + atomicOp!"-="(runningCount, 1); + } + + eventLoop.run(() => cancelled); + } + + private bool cancelled; + + void cancel() { + cancelled = true; + } + + EventLoopType type; + ICoreEventLoop eventLoop; + + __gshared static { + CoreWorkerThread[] taskRunners; + CoreWorkerThread[] helperRunners; + ICoreEventLoop mainThreadLoop; + + // for the helper function thing on the bsds i could have my own little circular buffer of availability + + shared(int) startedCount; + shared(int) runningCount; + + bool started; + + void setup(int numberOfTaskRunners, int numberOfHelpers) { + assert(!started); + synchronized { + mainThreadLoop = getThisThreadEventLoop(); + + foreach(i; 0 .. numberOfTaskRunners) { + auto nt = new CoreWorkerThread(EventLoopType.TaskRunner); + taskRunners ~= nt; + nt.start(); + } + foreach(i; 0 .. numberOfHelpers) { + auto nt = new CoreWorkerThread(EventLoopType.HelperWorker); + helperRunners ~= nt; + nt.start(); + } + + const expectedCount = numberOfHelpers + numberOfTaskRunners; + + while(startedCount < expectedCount) { + Thread.yield(); + } + + started = true; + } + } + + void cancelAll() { + foreach(runner; taskRunners) + runner.cancel(); + foreach(runner; helperRunners) + runner.cancel(); + + } + } +} + +private int numberOfCpus() { + return 4; // FIXME +} + +/++ + To opt in to the full functionality of this module with customization opportunity, create one and only one of these objects that is valid for exactly the lifetime of the application. + + Normally, this means writing a main like this: + + --- + import arsd.core; + void main() { + ArsdCoreApplication app = ArsdCoreApplication("Your app name"); + + // do your setup here + + // the rest of your code here + } + --- + + Its destructor runs the event loop then waits to for the workers to finish to clean them up. ++/ +// FIXME: single instance? +version(HasThread) struct ArsdCoreApplication { + private ICoreEventLoop impl; + + /++ + default number of threads is to split your cpus between blocking function runners and task runners + +/ + this(string applicationName) { + auto num = numberOfCpus(); + num /= 2; + if(num <= 0) + num = 1; + this(applicationName, num, num); + } + + /++ + + +/ + this(string applicationName, int numberOfTaskRunners, int numberOfHelpers) { + impl = getThisThreadEventLoop(EventLoopType.Explicit); + CoreWorkerThread.setup(numberOfTaskRunners, numberOfHelpers); + } + + @disable this(); + @disable this(this); + /++ + This must be deterministically destroyed. + +/ + @disable new(); + + ~this() { + if(!alreadyRun) + run(); + exitApplication(); + waitForWorkersToExit(3000); + } + + void exitApplication() { + CoreWorkerThread.cancelAll(); + } + + void waitForWorkersToExit(int timeoutMilliseconds) { + + } + + private bool alreadyRun; + + void run() { + impl.run(() => false); + alreadyRun = true; + } +} + + +private class CoreEventLoopImplementation : ICoreEventLoop { + version(EmptyEventLoop) RunOnceResult runOnce(Duration timeout = Duration.max) { return RunOnceResult(RunOnceResult.Possibilities.LocalExit); } + version(EmptyCoreEvent) + { + UnregisterToken addCallbackOnFdReadable(int fd, CallbackHelper cb){return typeof(return).init;} + RearmToken addCallbackOnFdReadableOneShot(int fd, CallbackHelper cb){return typeof(return).init;} + RearmToken addCallbackOnFdWritableOneShot(int fd, CallbackHelper cb){return typeof(return).init;} + private void rearmFd(RearmToken token) {} + } + + + private { + static struct LoopIterationDelegate { + void delegate() dg; + uint flags; + } + LoopIterationDelegate[] loopIterationDelegates; + + void runLoopIterationDelegates() { + foreach(lid; loopIterationDelegates) + lid.dg(); + } + } + + void addDelegateOnLoopIteration(void delegate() dg, uint timingFlags) { + loopIterationDelegates ~= LoopIterationDelegate(dg, timingFlags); + } + + version(Arsd_core_kqueue) { + // this thread apc dispatches go as a custom event to the queue + // the other queues go through one byte at a time pipes (barf). freebsd 13 and newest nbsd have eventfd too tho so maybe i can use them but the other kqueue systems don't. + + RunOnceResult runOnce(Duration timeout = Duration.max) { + scope(exit) eventLoopRound++; + kevent_t[16] ev; + //timespec tout = timespec(1, 0); + auto nev = kevent(kqueuefd, null, 0, ev.ptr, ev.length, null/*&tout*/); + if(nev == -1) { + // FIXME: EINTR + throw new SystemApiException("kevent", errno); + } else if(nev == 0) { + // timeout + } else { + foreach(event; ev[0 .. nev]) { + if(event.filter == EVFILT_SIGNAL) { + // FIXME: I could prolly do this better tbh + markSignalOccurred(cast(int) event.ident); + signalChecker(); + } else { + // FIXME: event.filter more specific? + CallbackHelper cb = cast(CallbackHelper) event.udata; + cb.call(); + } + } + } + + runLoopIterationDelegates(); + + return RunOnceResult(RunOnceResult.Possibilities.CarryOn); + } + + // FIXME: idk how to make one event that multiple kqueues can listen to w/o being shared + // maybe a shared kqueue could work that the thread kqueue listen to (which i rejected for + // epoll cuz it caused thundering herd problems but maybe it'd work here) + + UnregisterToken addCallbackOnFdReadable(int fd, CallbackHelper cb) { + kevent_t ev; + + EV_SET(&ev, fd, EVFILT_READ, EV_ADD | EV_ENABLE/* | EV_ONESHOT*/, 0, 0, cast(void*) cb); + + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + + return UnregisterToken(this, fd, cb); + } + + RearmToken addCallbackOnFdReadableOneShot(int fd, CallbackHelper cb) { + kevent_t ev; + + EV_SET(&ev, fd, EVFILT_READ, EV_ADD | EV_ENABLE/* | EV_ONESHOT*/, 0, 0, cast(void*) cb); + + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + + return RearmToken(true, this, fd, cb, 0); + } + + RearmToken addCallbackOnFdWritableOneShot(int fd, CallbackHelper cb) { + kevent_t ev; + + EV_SET(&ev, fd, EVFILT_WRITE, EV_ADD | EV_ENABLE/* | EV_ONESHOT*/, 0, 0, cast(void*) cb); + + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + + return RearmToken(false, this, fd, cb, 0); + } + + private void rearmFd(RearmToken token) { + if(token.readable) + cast(void) addCallbackOnFdReadableOneShot(token.fd, token.cb); + else + cast(void) addCallbackOnFdWritableOneShot(token.fd, token.cb); + } + + private void triggerGlobalEvent() { + ubyte a; + import core.sys.posix.unistd; + write(kqueueGlobalFd[1], &a, 1); + } + + private this() { + kqueuefd = ErrnoEnforce!kqueue(); + setCloExec(kqueuefd); // FIXME O_CLOEXEC + + if(kqueueGlobalFd[0] == 0) { + import core.sys.posix.unistd; + pipe(kqueueGlobalFd); + setCloExec(kqueueGlobalFd[0]); + setCloExec(kqueueGlobalFd[1]); + + signal(SIGINT, SIG_IGN); // FIXME + } + + kevent_t ev; + + EV_SET(&ev, SIGCHLD, EVFILT_SIGNAL, EV_ADD | EV_ENABLE, 0, 0, null); + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + EV_SET(&ev, SIGINT, EVFILT_SIGNAL, EV_ADD | EV_ENABLE, 0, 0, null); + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + + globalEventSent = new CallbackHelper(&readGlobalEvent); + EV_SET(&ev, kqueueGlobalFd[0], EVFILT_READ, EV_ADD | EV_ENABLE, 0, 0, cast(void*) globalEventSent); + ErrnoEnforce!kevent(kqueuefd, &ev, 1, null, 0, null); + } + + private int kqueuefd = -1; + + private CallbackHelper globalEventSent; + void readGlobalEvent() { + kevent_t event; + + import core.sys.posix.unistd; + ubyte a; + read(kqueueGlobalFd[0], &a, 1); + + // FIXME: the thread is woken up, now we need to check the circualr buffer queue + } + + private __gshared int[2] kqueueGlobalFd; + } + + /+ + // this setup needs no extra allocation + auto op = read(file, buffer); + op.oncomplete = &thisfiber.call; + op.start(); + thisfiber.yield(); + auto result = op.waitForCompletion(); // guaranteed to return instantly thanks to previous setup + + can generically abstract that into: + + auto result = thisTask.await(read(file, buffer)); + + + You MUST NOT use buffer in any way - not read, modify, deallocate, reuse, anything - until the PendingOperation is complete. + + Note that PendingOperation may just be a wrapper around an internally allocated object reference... but then if you do a waitForFirstToComplete what happens? + + those could of course just take the value type things + +/ + + + version(Arsd_core_windows) { + // all event loops share the one iocp, Windows + // manages how to do it + __gshared HANDLE iocpTaskRunners; + __gshared HANDLE iocpWorkers; + + HANDLE[] handles; + CallbackHelper[] handlesCbs; + + void unregisterHandle(HANDLE handle, CallbackHelper cb) { + foreach(idx, h; handles) + if(h is handle && handlesCbs[idx] is cb) { + handles[idx] = handles[$-1]; + handles = handles[0 .. $-1].assumeSafeAppend; + + handlesCbs[idx] = handlesCbs[$-1]; + handlesCbs = handlesCbs[0 .. $-1].assumeSafeAppend; + } + } + + UnregisterToken addCallbackOnHandleReady(HANDLE handle, CallbackHelper cb) { + handles ~= handle; + handlesCbs ~= cb; + + return UnregisterToken(this, handle, cb); + } + + // i think to terminate i just have to post the message at least once for every thread i know about, maybe a few more times for threads i don't know about. + + bool isWorker; // if it is a worker we wait on the iocp, if not we wait on msg + + RunOnceResult runOnce(Duration timeout = Duration.max) { + scope(exit) eventLoopRound++; + if(isWorker) { + // this function is only supported on Windows Vista and up, so using this + // means dropping support for XP. + //GetQueuedCompletionStatusEx(); + assert(0); // FIXME + } else { + auto wto = 0; + + auto waitResult = MsgWaitForMultipleObjectsEx( + cast(int) handles.length, handles.ptr, + (wto == 0 ? INFINITE : wto), /* timeout */ + 0x04FF, /* QS_ALLINPUT */ + 0x0002 /* MWMO_ALERTABLE */ | 0x0004 /* MWMO_INPUTAVAILABLE */); + + enum WAIT_OBJECT_0 = 0; + if(waitResult >= WAIT_OBJECT_0 && waitResult < handles.length + WAIT_OBJECT_0) { + auto h = handles[waitResult - WAIT_OBJECT_0]; + auto cb = handlesCbs[waitResult - WAIT_OBJECT_0]; + cb.call(); + } else if(waitResult == handles.length + WAIT_OBJECT_0) { + // message ready + int count; + MSG message; + while(PeekMessage(&message, null, 0, 0, PM_NOREMOVE)) { // need to peek since sometimes MsgWaitForMultipleObjectsEx returns even though GetMessage can block. tbh i don't fully understand it but the docs say it is foreground activation + auto ret = GetMessage(&message, null, 0, 0); + if(ret == -1) + throw new WindowsApiException("GetMessage", GetLastError()); + TranslateMessage(&message); + DispatchMessage(&message); + + count++; + if(count > 10) + break; // take the opportunity to catch up on other events + + if(ret == 0) { // WM_QUIT + exitApplication(); + } + } + } else if(waitResult == 0x000000C0L /* WAIT_IO_COMPLETION */) { + SleepEx(0, true); // I call this to give it a chance to do stuff like async io + } else if(waitResult == 258L /* WAIT_TIMEOUT */) { + // timeout, should never happen since we aren't using it + } else if(waitResult == 0xFFFFFFFF) { + // failed + throw new WindowsApiException("MsgWaitForMultipleObjectsEx", GetLastError()); + } else { + // idk.... + } + } + + runLoopIterationDelegates(); + + return RunOnceResult(RunOnceResult.Possibilities.CarryOn); + } + } + + version(Posix) { + private __gshared uint sigChildHappened = 0; + private __gshared uint sigIntrHappened = 0; + + static void signalChecker() { + if(cas(&sigChildHappened, 1, 0)) { + while(true) { // multiple children could have exited before we processed the notification + + import core.sys.posix.sys.wait; + + int status; + auto pid = waitpid(-1, &status, WNOHANG); + if(pid == -1) { + import core.stdc.errno; + auto errno = errno; + if(errno == ECHILD) + break; // also all done, there are no children left + // no need to check EINTR since we set WNOHANG + throw new ErrnoApiException("waitpid", errno); + } + if(pid == 0) + break; // all done, all children are still running + + // look up the pid for one of our objects + // if it is found, inform it of its status + // and then inform its controlling thread + // to wake up so it can check its waitForCompletion, + // trigger its callbacks, etc. + + ExternalProcess.recordChildTerminated(pid, status); + } + + } + if(cas(&sigIntrHappened, 1, 0)) { + // FIXME + import core.stdc.stdlib; + exit(0); + } + } + + /++ + Informs the arsd.core system that the given signal happened. You can call this from inside a signal handler. + +/ + public static void markSignalOccurred(int sigNumber) nothrow { + import core.sys.posix.unistd; + + if(sigNumber == SIGCHLD) + volatileStore(&sigChildHappened, 1); + if(sigNumber == SIGINT) + volatileStore(&sigIntrHappened, 1); + + version(Arsd_core_epoll) { + ulong writeValue = 1; + write(signalPipeFd, &writeValue, writeValue.sizeof); + } + } + } + + version(Arsd_core_epoll) { + + import core.sys.linux.epoll; + import core.sys.linux.sys.eventfd; + + private this() { + + if(!globalsInitialized) { + synchronized { + if(!globalsInitialized) { + // blocking signals is problematic because it is inherited by child processes + // and that can be problematic for general purpose stuff so i use a self pipe + // here. though since it is linux, im using an eventfd instead just to notify + signalPipeFd = ErrnoEnforce!eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + signalReaderCallback = new CallbackHelper(&signalReader); + + runInTaskRunnerQueue = new CallbackQueue("task runners", true); + runInHelperThreadQueue = new CallbackQueue("helper threads", true); + + setSignalHandlers(); + + globalsInitialized = true; + } + } + } + + epollfd = epoll_create1(EPOLL_CLOEXEC); + + // FIXME: ensure UI events get top priority + + // global listeners + + // FIXME: i should prolly keep the tokens and release them when tearing down. + + cast(void) addCallbackOnFdReadable(signalPipeFd, signalReaderCallback); + if(true) { // FIXME: if this is a task runner vs helper thread vs ui thread + cast(void) addCallbackOnFdReadable(runInTaskRunnerQueue.fd, runInTaskRunnerQueue.callback); + runInTaskRunnerQueue.callback.addref(); + } else { + cast(void) addCallbackOnFdReadable(runInHelperThreadQueue.fd, runInHelperThreadQueue.callback); + runInHelperThreadQueue.callback.addref(); + } + + // local listener + thisThreadQueue = new CallbackQueue("this thread", false); + cast(void) addCallbackOnFdReadable(thisThreadQueue.fd, thisThreadQueue.callback); + + // what are we going to do about timers? + } + + void teardown() { + import core.sys.posix.fcntl; + import core.sys.posix.unistd; + + close(epollfd); + epollfd = -1; + + thisThreadQueue.teardown(); + + // FIXME: should prolly free anything left in the callback queue, tho those could also be GC managed tbh. + } + + /+ // i call it explicitly at the thread exit instead, but worker threads aren't really supposed to exit generally speaking till process done anyway + static ~this() { + teardown(); + } + +/ + + static void teardownGlobals() { + import core.sys.posix.fcntl; + import core.sys.posix.unistd; + + synchronized { + restoreSignalHandlers(); + close(signalPipeFd); + signalReaderCallback.release(); + + runInTaskRunnerQueue.teardown(); + runInHelperThreadQueue.teardown(); + + globalsInitialized = false; + } + + } + + + private static final class CallbackQueue { + int fd = -1; + string name; + CallbackHelper callback; + SynchronizedCircularBuffer!CallbackHelper queue; + + this(string name, bool dequeueIsShared) { + this.name = name; + queue = typeof(queue)(this); + + fd = ErrnoEnforce!eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK | (dequeueIsShared ? EFD_SEMAPHORE : 0)); + + callback = new CallbackHelper(dequeueIsShared ? &sharedDequeueCb : &threadLocalDequeueCb); + } + + bool resetEvent() { + import core.sys.posix.unistd; + ulong count; + return read(fd, &count, count.sizeof) == count.sizeof; + } + + void sharedDequeueCb() { + if(resetEvent()) { + auto cb = queue.dequeue(); + cb.call(); + cb.release(); + } + } + + void threadLocalDequeueCb() { + CallbackHelper[16] buffer; + foreach(cb; queue.dequeueSeveral(buffer[], () { resetEvent(); })) { + cb.call(); + cb.release(); + } + } + + void enqueue(CallbackHelper cb) { + if(queue.enqueue(cb)) { + import core.sys.posix.unistd; + ulong count = 1; + ErrnoEnforce!write(fd, &count, count.sizeof); + } else { + throw new ArsdException!"queue is full"(name); + } + } + + void teardown() { + import core.sys.posix.fcntl; + import core.sys.posix.unistd; + + close(fd); + fd = -1; + + callback.release(); + } + } + + // there's a global instance of this we refer back to + private __gshared { + bool globalsInitialized; + + CallbackHelper signalReaderCallback; + + CallbackQueue runInTaskRunnerQueue; + CallbackQueue runInHelperThreadQueue; + + int exitEventFd = -1; // FIXME: implement + } + + // and then the local loop + private { + int epollfd = -1; + + CallbackQueue thisThreadQueue; + } + + // signal stuff { + import core.sys.posix.signal; + + private __gshared sigaction_t oldSigIntr; + private __gshared sigaction_t oldSigChld; + private __gshared sigaction_t oldSigPipe; + + private __gshared int signalPipeFd = -1; + // sigpipe not important, i handle errors on the writes + + public static void setSignalHandlers() { + static extern(C) void interruptHandler(int sigNumber) nothrow { + markSignalOccurred(sigNumber); + + /+ + // calling the old handler is non-trivial since there can be ignore + // or default or a plain handler or a sigaction 3 arg handler and i + // i don't think it is worth teh complication + sigaction_t* oldHandler; + if(sigNumber == SIGCHLD) + oldHandler = &oldSigChld; + else if(sigNumber == SIGINT) + oldHandler = &oldSigIntr; + if(oldHandler && oldHandler.sa_handler) + oldHandler + +/ + } + + sigaction_t n; + n.sa_handler = &interruptHandler; + n.sa_mask = cast(sigset_t) 0; + n.sa_flags = 0; + sigaction(SIGINT, &n, &oldSigIntr); + sigaction(SIGCHLD, &n, &oldSigChld); + + n.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &n, &oldSigPipe); + } + + public static void restoreSignalHandlers() { + sigaction(SIGINT, &oldSigIntr, null); + sigaction(SIGCHLD, &oldSigChld, null); + sigaction(SIGPIPE, &oldSigPipe, null); + } + + private static void signalReader() { + import core.sys.posix.unistd; + ulong number; + read(signalPipeFd, &number, number.sizeof); + + signalChecker(); + } + // signal stuff done } + + // the any thread poll is just registered in the this thread poll w/ exclusive. nobody actaully epoll_waits + // on the global one directly. + + RunOnceResult runOnce(Duration timeout = Duration.max) { + scope(exit) eventLoopRound++; + epoll_event[16] events; + auto ret = epoll_wait(epollfd, events.ptr, cast(int) events.length, -1); // FIXME: timeout + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) { + return RunOnceResult(RunOnceResult.Possibilities.Interrupted); + } + throw new ErrnoApiException("epoll_wait", errno); + } else if(ret == 0) { + // timeout + } else { + // loop events and call associated callbacks + foreach(event; events[0 .. ret]) { + auto flags = event.events; + auto cbObject = cast(CallbackHelper) event.data.ptr; + + // FIXME: or if it is an error... + // EPOLLERR - write end of pipe when read end closed or other error. and EPOLLHUP - terminal hangup or read end when write end close (but it will give 0 reading after that soon anyway) + + cbObject.call(); + } + } + + runLoopIterationDelegates(); + + return RunOnceResult(RunOnceResult.Possibilities.CarryOn); + } + + // building blocks for low-level integration with the loop + + UnregisterToken addCallbackOnFdReadable(int fd, CallbackHelper cb) { + epoll_event event; + event.data.ptr = cast(void*) cb; + event.events = EPOLLIN | EPOLLEXCLUSIVE; + if(epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event) == -1) + throw new ErrnoApiException("epoll_ctl", errno); + + return UnregisterToken(this, fd, cb); + } + + /++ + Adds a one-off callback that you can optionally rearm when it happens. + +/ + RearmToken addCallbackOnFdReadableOneShot(int fd, CallbackHelper cb) { + epoll_event event; + event.data.ptr = cast(void*) cb; + event.events = EPOLLIN | EPOLLONESHOT; + if(epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event) == -1) + throw new ErrnoApiException("epoll_ctl", errno); + + return RearmToken(true, this, fd, cb, EPOLLIN | EPOLLONESHOT); + } + + /++ + Adds a one-off callback that you can optionally rearm when it happens. + +/ + RearmToken addCallbackOnFdWritableOneShot(int fd, CallbackHelper cb) { + epoll_event event; + event.data.ptr = cast(void*) cb; + event.events = EPOLLOUT | EPOLLONESHOT; + if(epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &event) == -1) + throw new ErrnoApiException("epoll_ctl", errno); + + return RearmToken(false, this, fd, cb, EPOLLOUT | EPOLLONESHOT); + } + + private void unregisterFd(int fd) { + epoll_event event; + if(epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, &event) == -1) + throw new ErrnoApiException("epoll_ctl", errno); + } + + private void rearmFd(RearmToken token) { + epoll_event event; + event.data.ptr = cast(void*) token.cb; + event.events = token.flags; + if(epoll_ctl(epollfd, EPOLL_CTL_MOD, token.fd, &event) == -1) + throw new ErrnoApiException("epoll_ctl", errno); + } + + // Disk files will have to be sent as messages to a worker to do the read and report back a completion packet. + } + + version(Arsd_core_kqueue) { + // FIXME + } + + // cross platform adapters + void setTimeout() {} + void addFileOrDirectoryChangeListener(FilePath name, uint flags, bool recursive = false) {} +} + +// deduplication???????// +bool postMessage(ThreadToRunIn destination, void delegate() code) { + return false; +} +bool postMessage(ThreadToRunIn destination, Object message) { + return false; +} + +/+ +void main() { + // FIXME: the offset doesn't seem to be done right + auto file = new AsyncFile(FilePath("test.txt"), AsyncFile.OpenMode.writeWithTruncation); + file.write("hello", 10).waitForCompletion(); +} ++/ + +// to test the mailboxes +/+ +void main() { + /+ + import std.stdio; + Thread[4] pool; + + bool shouldExit; + + static int received; + + static void tester() { + received++; + //writeln(cast(void*) Thread.getThis, " ", received); + } + + foreach(ref thread; pool) { + thread = new Thread(() { + getThisThreadEventLoop().run(() { + return shouldExit; + }); + }); + thread.start(); + } + + getThisThreadEventLoop(); // ensure it is all initialized before proceeding. FIXME: i should have an ensure initialized function i do on most the public apis. + + int lol; + + try + foreach(i; 0 .. 6000) { + CoreEventLoopImplementation.runInTaskRunnerQueue.enqueue(new CallbackHelper(&tester)); + lol = cast(int) i; + } + catch(ArsdExceptionBase e) { + Thread.sleep(50.msecs); + writeln(e); + writeln(lol); + } + + import core.stdc.stdlib; + exit(0); + + version(none) + foreach(i; 0 .. 100) + CoreEventLoopImplementation.runInTaskRunnerQueue.enqueue(new CallbackHelper(&tester)); + + + foreach(ref thread; pool) { + thread.join(); + } + +/ + + + static int received; + + static void tester() { + received++; + //writeln(cast(void*) Thread.getThis, " ", received); + } + + + + auto ev = cast(CoreEventLoopImplementation) getThisThreadEventLoop(); + foreach(i; 0 .. 100) + ev.thisThreadQueue.enqueue(new CallbackHelper(&tester)); + foreach(i; 0 .. 100 / 16 + 1) + ev.runOnce(); + import std.conv; + assert(received == 100, to!string(received)); + +} ++/ + +/++ + This is primarily a helper for the event queues. It is public in the hope it might be useful, + but subject to change without notice; I will treat breaking it the same as if it is private. + (That said, it is a simple little utility that does its job, so it is unlikely to change much. + The biggest change would probably be letting it grow and changing from inline to dynamic array.) + + It is a fixed-size ring buffer that synchronizes on a given object you give it in the constructor. + + After enqueuing something, you should probably set an event to notify the other threads. This is left + as an exercise to you (or another wrapper). ++/ +struct SynchronizedCircularBuffer(T, size_t maxSize = 128) { + private T[maxSize] ring; + private int front; + private int back; + + private Object synchronizedOn; + + @disable this(); + + /++ + The Object's monitor is used to synchronize the methods in here. + +/ + this(Object synchronizedOn) { + this.synchronizedOn = synchronizedOn; + } + + /++ + Note the potential race condition between calling this and actually dequeuing something. You might + want to acquire the lock on the object before calling this (nested synchronized things are allowed + as long as the same thread is the one doing it). + +/ + bool isEmpty() { + synchronized(this.synchronizedOn) { + return front == back; + } + } + + /++ + Note the potential race condition between calling this and actually queuing something. + +/ + bool isFull() { + synchronized(this.synchronizedOn) { + return isFullUnsynchronized(); + } + } + + private bool isFullUnsynchronized() nothrow const { + return ((back + 1) % ring.length) == front; + + } + + /++ + If this returns true, you should signal listening threads (with an event or a semaphore, + depending on how you dequeue it). If it returns false, the queue was full and your thing + was NOT added. You might wait and retry later (you could set up another event to signal it + has been read and wait for that, or maybe try on a timer), or just fail and throw an exception + or to abandon the message. + +/ + bool enqueue(T what) { + synchronized(this.synchronizedOn) { + if(isFullUnsynchronized()) + return false; + ring[(back++) % ring.length] = what; + return true; + } + } + + private T dequeueUnsynchronized() nothrow { + assert(front != back); + return ring[(front++) % ring.length]; + } + + /++ + If you are using a semaphore to signal, you can call this once for each count of it + and you can do that separately from this call (though they should be paired). + + If you are using an event, you should use [dequeueSeveral] instead to drain it. + +/ + T dequeue() { + synchronized(this.synchronizedOn) { + return dequeueUnsynchronized(); + } + } + + /++ + Note that if you use a semaphore to signal waiting threads, you should probably not call this. + + If you use a set/reset event, there's a potential race condition between the dequeue and event + reset. This is why the `runInsideLockIfEmpty` delegate is there - when it is empty, before it + unlocks, it will give you a chance to reset the event. Otherwise, it can remain set to indicate + that there's still pending data in the queue. + +/ + T[] dequeueSeveral(return T[] buffer, scope void delegate() runInsideLockIfEmpty = null) { + int pos; + synchronized(this.synchronizedOn) { + while(pos < buffer.length && front != back) { + buffer[pos++] = dequeueUnsynchronized(); + } + if(front == back && runInsideLockIfEmpty !is null) + runInsideLockIfEmpty(); + } + return buffer[0 .. pos]; + } +} + +unittest { + Object object = new Object(); + auto queue = SynchronizedCircularBuffer!CallbackHelper(object); + assert(queue.isEmpty); + foreach(i; 0 .. queue.ring.length - 1) + queue.enqueue(cast(CallbackHelper) cast(void*) i); + assert(queue.isFull); + + foreach(i; 0 .. queue.ring.length - 1) + assert(queue.dequeue() is (cast(CallbackHelper) cast(void*) i)); + assert(queue.isEmpty); + + foreach(i; 0 .. queue.ring.length - 1) + queue.enqueue(cast(CallbackHelper) cast(void*) i); + assert(queue.isFull); + + CallbackHelper[] buffer = new CallbackHelper[](300); + auto got = queue.dequeueSeveral(buffer); + assert(got.length == queue.ring.length - 1); + assert(queue.isEmpty); + foreach(i, item; got) + assert(item is (cast(CallbackHelper) cast(void*) i)); + + foreach(i; 0 .. 8) + queue.enqueue(cast(CallbackHelper) cast(void*) i); + buffer = new CallbackHelper[](4); + got = queue.dequeueSeveral(buffer); + assert(got.length == 4); + foreach(i, item; got) + assert(item is (cast(CallbackHelper) cast(void*) i)); + got = queue.dequeueSeveral(buffer); + assert(got.length == 4); + foreach(i, item; got) + assert(item is (cast(CallbackHelper) cast(void*) (i+4))); + got = queue.dequeueSeveral(buffer); + assert(got.length == 0); + assert(queue.isEmpty); +} + +/++ + ++/ +enum ByteOrder { + irrelevant, + littleEndian, + bigEndian, +} + +/++ + A class to help write a stream of binary data to some target. + + NOT YET FUNCTIONAL ++/ +class WritableStream { + /++ + + +/ + this(size_t bufferSize) { + this(new ubyte[](bufferSize)); + } + + /// ditto + this(ubyte[] buffer) { + this.buffer = buffer; + } + + /++ + + +/ + final void put(T)(T value, ByteOrder byteOrder = ByteOrder.irrelevant, string file = __FILE__, size_t line = __LINE__) { + static if(T.sizeof == 8) + ulong b; + else static if(T.sizeof == 4) + uint b; + else static if(T.sizeof == 2) + ushort b; + else static if(T.sizeof == 1) + ubyte b; + else static assert(0, "unimplemented type, try using just the basic types"); + + if(byteOrder == ByteOrder.irrelevant && T.sizeof > 1) + throw new InvalidArgumentsException("byteOrder", "byte order must be specified for type " ~ T.stringof ~ " because it is bigger than one byte", "WritableStream.put", file, line); + + final switch(byteOrder) { + case ByteOrder.irrelevant: + writeOneByte(b); + break; + case ByteOrder.littleEndian: + foreach(i; 0 .. T.sizeof) { + writeOneByte(b & 0xff); + b >>= 8; + } + break; + case ByteOrder.bigEndian: + int amount = T.sizeof * 8 - 8; + foreach(i; 0 .. T.sizeof) { + writeOneByte((b >> amount) & 0xff); + amount -= 8; + } + break; + } + } + + /// ditto + final void put(T : E[], E)(T value, ByteOrder elementByteOrder = ByteOrder.irrelevant, string file = __FILE__, size_t line = __LINE__) { + foreach(item; value) + put(item, elementByteOrder, file, line); + } + + /++ + Performs a final flush() call, then marks the stream as closed, meaning no further data will be written to it. + +/ + void close() { + isClosed_ = true; + } + + /++ + Writes what is currently in the buffer to the target and waits for the target to accept it. + Please note: if you are subclassing this to go to a different target + +/ + void flush() {} + + /++ + Returns true if either you closed it or if the receiving end closed their side, indicating they + don't want any more data. + +/ + bool isClosed() { + return isClosed_; + } + + // hasRoomInBuffer + // canFlush + // waitUntilCanFlush + + // flushImpl + // markFinished / close - tells the other end you're done + + private final writeOneByte(ubyte value) { + if(bufferPosition == buffer.length) + flush(); + + buffer[bufferPosition++] = value; + } + + + private { + ubyte[] buffer; + int bufferPosition; + bool isClosed_; + } +} + +/++ + A stream can be used by just one task at a time, but one task can consume multiple streams. + + Streams may be populated by async sources (in which case they must be called from a fiber task), + from a function generating the data on demand (including an input range), from memory, or from a synchronous file. + + A stream of heterogeneous types is compatible with input ranges. + + It reads binary data. ++/ +version(HasThread) class ReadableStream { + + this() { + + } + + /++ + Gets data of the specified type `T` off the stream. The byte order of the T on the stream must be specified unless it is irrelevant (e.g. single byte entries). + + --- + // get an int out of a big endian stream + int i = stream.get!int(ByteOrder.bigEndian); + + // get i bytes off the stream + ubyte[] data = stream.get!(ubyte[])(i); + --- + +/ + final T get(T)(ByteOrder byteOrder = ByteOrder.irrelevant, string file = __FILE__, size_t line = __LINE__) { + if(byteOrder == ByteOrder.irrelevant && T.sizeof > 1) + throw new InvalidArgumentsException("byteOrder", "byte order must be specified for type " ~ T.stringof ~ " because it is bigger than one byte", "ReadableStream.get", file, line); + + // FIXME: what if it is a struct? + + while(bufferedLength() < T.sizeof) + waitForAdditionalData(); + + static if(T.sizeof == 1) { + ubyte ret = consumeOneByte(); + return *cast(T*) &ret; + } else { + static if(T.sizeof == 8) + ulong ret; + else static if(T.sizeof == 4) + uint ret; + else static if(T.sizeof == 2) + ushort ret; + else static assert(0, "unimplemented type, try using just the basic types"); + + if(byteOrder == ByteOrder.littleEndian) { + typeof(ret) buffer; + foreach(b; 0 .. T.sizeof) { + buffer = consumeOneByte(); + buffer <<= T.sizeof * 8 - 8; + + ret >>= 8; + ret |= buffer; + } + } else { + foreach(b; 0 .. T.sizeof) { + ret <<= 8; + ret |= consumeOneByte(); + } + } + + return *cast(T*) &ret; + } + } + + /// ditto + final T get(T : E[], E)(size_t length, ByteOrder elementByteOrder = ByteOrder.irrelevant, string file = __FILE__, size_t line = __LINE__) { + if(elementByteOrder == ByteOrder.irrelevant && E.sizeof > 1) + throw new InvalidArgumentsException("elementByteOrder", "byte order must be specified for type " ~ E.stringof ~ " because it is bigger than one byte", "ReadableStream.get", file, line); + + // if the stream is closed before getting the length or the terminator, should we send partial stuff + // or just throw? + + while(bufferedLength() < length * E.sizeof) + waitForAdditionalData(); + + T ret; + + ret.length = length; + + if(false && elementByteOrder == ByteOrder.irrelevant) { + // ret[] = + // FIXME: can prolly optimize + } else { + foreach(i; 0 .. length) + ret[i] = get!E(elementByteOrder); + } + + return ret; + + } + + /// ditto + final T get(T : E[], E)(scope bool delegate(E e) isTerminatingSentinel, ByteOrder elementByteOrder = ByteOrder.irrelevant, string file = __FILE__, size_t line = __LINE__) { + if(byteOrder == ByteOrder.irrelevant && E.sizeof > 1) + throw new InvalidArgumentsException("elementByteOrder", "byte order must be specified for type " ~ E.stringof ~ " because it is bigger than one byte", "ReadableStream.get", file, line); + + assert(0, "Not implemented"); + } + + /++ + + +/ + bool isClosed() { + return isClosed_; + } + + // Control side of things + + private bool isClosed_; + + /++ + Feeds data into the stream, which can be consumed by `get`. If a task is waiting for more + data to satisfy its get requests, this will trigger those tasks to resume. + + If you feed it empty data, it will mark the stream as closed. + +/ + void feedData(ubyte[] data) { + if(data.length == 0) + isClosed_ = true; + + currentBuffer = data; + // this is a borrowed buffer, so we won't keep the reference long term + scope(exit) + currentBuffer = null; + + if(waitingTask !is null) { + waitingTask.call(); + } + } + + /++ + You basically have to use this thing from a task + +/ + protected void waitForAdditionalData() { + Fiber task = Fiber.getThis; + + assert(task !is null); + + if(waitingTask !is null && waitingTask !is task) + throw new ArsdException!"streams can only have one waiting task"; + + // copy any pending data in our buffer to the longer-term buffer + if(currentBuffer.length) + leftoverBuffer ~= currentBuffer; + + waitingTask = task; + task.yield(); + } + + private Fiber waitingTask; + private ubyte[] leftoverBuffer; + private ubyte[] currentBuffer; + + private size_t bufferedLength() { + return leftoverBuffer.length + currentBuffer.length; + } + + private ubyte consumeOneByte() { + ubyte b; + if(leftoverBuffer.length) { + b = leftoverBuffer[0]; + leftoverBuffer = leftoverBuffer[1 .. $]; + } else if(currentBuffer.length) { + b = currentBuffer[0]; + currentBuffer = currentBuffer[1 .. $]; + } else { + assert(0, "consuming off an empty buffer is impossible"); + } + + return b; + } +} + +// FIXME: do a stringstream too + +unittest { + auto stream = new ReadableStream(); + + int position; + char[16] errorBuffer; + + auto fiber = new Fiber(() { + position = 1; + int a = stream.get!int(ByteOrder.littleEndian); + assert(a == 10, intToString(a, errorBuffer[])); + position = 2; + ubyte b = stream.get!ubyte; + assert(b == 33); + position = 3; + + // ubyte[] c = stream.get!(ubyte[])(3); + // int[] d = stream.get!(int[])(3); + }); + + fiber.call(); + assert(position == 1); + stream.feedData([10, 0, 0, 0]); + assert(position == 2); + stream.feedData([33]); + assert(position == 3); + + // stream.feedData([1,2,3]); + // stream.feedData([1,2,3,4,1,2,3,4,1,2,3,4]); +} + +/++ + UNSTABLE, NOT FULLY IMPLEMENTED. DO NOT USE YET. + + You might use this like: + + --- + auto proc = new ExternalProcess(); + auto stdoutStream = new ReadableStream(); + + // to use a stream you can make one and have a task consume it + runTask({ + while(!stdoutStream.isClosed) { + auto line = stdoutStream.get!string(e => e == '\n'); + } + }); + + // then make the process feed into the stream + proc.onStdoutAvailable = (got) { + stdoutStream.feedData(got); // send it to the stream for processing + stdout.rawWrite(got); // forward it through to our own thing + // could also append it to a buffer to return it on complete + }; + proc.start(); + --- + + Please note that this does not currently and I have no plans as of this writing to add support for any kind of direct file descriptor passing. It always pipes them back to the parent for processing. If you don't want this, call the lower level functions yourself; the reason this class is here is to aid integration in the arsd.core event loop. Of course, I might change my mind on this. + + Bugs: + Not implemented at all on Windows yet. ++/ +class ExternalProcess /*: AsyncOperationRequest*/ { + + private static version(Posix) { + __gshared ExternalProcess[pid_t] activeChildren; + + void recordChildCreated(pid_t pid, ExternalProcess proc) { + synchronized(typeid(ExternalProcess)) { + activeChildren[pid] = proc; + } + } + + void recordChildTerminated(pid_t pid, int status) { + synchronized(typeid(ExternalProcess)) { + if(pid in activeChildren) { + auto ac = activeChildren[pid]; + ac.completed = true; + ac.status = status; + activeChildren.remove(pid); + } + } + } + } + + // FIXME: config to pass through a shell or not + + /++ + This is the native version for Windows. + +/ + this(string program, string commandLine) { + version(Posix) { + assert(0, "not implemented command line to posix args yet"); + } + else throw new NotYetImplementedException(); + } + + this(string commandLine) { + version(Posix) { + assert(0, "not implemented command line to posix args yet"); + } + else throw new NotYetImplementedException(); + } + + this(string[] args) { + version(Posix) { + this.program = FilePath(args[0]); + this.args = args; + } + else throw new NotYetImplementedException(); + } + + /++ + This is the native version for Posix. + +/ + this(FilePath program, string[] args) { + version(Posix) { + this.program = program; + this.args = args; + } + else throw new NotYetImplementedException(); + } + + // you can modify these before calling start + int stdoutBufferSize = 32 * 1024; + int stderrBufferSize = 8 * 1024; + + void start() { + version(Posix) { + int ret; + + int[2] stdinPipes; + ret = pipe(stdinPipes); + if(ret == -1) + throw new ErrnoApiException("stdin pipe", errno); + + scope(failure) { + close(stdinPipes[0]); + close(stdinPipes[1]); + } + + stdinFd = stdinPipes[1]; + + int[2] stdoutPipes; + ret = pipe(stdoutPipes); + if(ret == -1) + throw new ErrnoApiException("stdout pipe", errno); + + scope(failure) { + close(stdoutPipes[0]); + close(stdoutPipes[1]); + } + + stdoutFd = stdoutPipes[0]; + + int[2] stderrPipes; + ret = pipe(stderrPipes); + if(ret == -1) + throw new ErrnoApiException("stderr pipe", errno); + + scope(failure) { + close(stderrPipes[0]); + close(stderrPipes[1]); + } + + stderrFd = stderrPipes[0]; + + + int[2] errorReportPipes; + ret = pipe(errorReportPipes); + if(ret == -1) + throw new ErrnoApiException("error reporting pipe", errno); + + scope(failure) { + close(errorReportPipes[0]); + close(errorReportPipes[1]); + } + + setCloExec(errorReportPipes[0]); + setCloExec(errorReportPipes[1]); + + auto forkRet = fork(); + if(forkRet == -1) + throw new ErrnoApiException("fork", errno); + + if(forkRet == 0) { + // child side + + // FIXME can we do more error checking that is actually useful here? + // these operations are virtually guaranteed to succeed given the setup anyway. + + // FIXME pty too + + void fail(int step) { + import core.stdc.errno; + auto code = errno; + + // report the info back to the parent then exit + + int[2] msg = [step, code]; + auto ret = write(errorReportPipes[1], msg.ptr, msg.sizeof); + + // but if this fails there's not much we can do... + + import core.stdc.stdlib; + exit(1); + } + + // dup2 closes the fd it is replacing automatically + dup2(stdinPipes[0], 0); + dup2(stdoutPipes[1], 1); + dup2(stderrPipes[1], 2); + + // don't need either of the original pipe fds anymore + close(stdinPipes[0]); + close(stdinPipes[1]); + close(stdoutPipes[0]); + close(stdoutPipes[1]); + close(stderrPipes[0]); + close(stderrPipes[1]); + + // the error reporting pipe will be closed upon exec since we set cloexec before fork + // and everything else should have cloexec set too hopefully. + + if(beforeExec) + beforeExec(); + + // i'm not sure that a fully-initialized druntime is still usable + // after a fork(), so i'm gonna stick to the C lib in here. + + const(char)* file = mallocedStringz(program.path).ptr; + if(file is null) + fail(1); + const(char)*[] argv = mallocSlice!(const(char)*)(args.length + 1); + if(argv is null) + fail(2); + foreach(idx, arg; args) { + argv[idx] = mallocedStringz(args[idx]).ptr; + if(argv[idx] is null) + fail(3); + } + argv[args.length] = null; + + auto rete = execvp/*e*/(file, argv.ptr/*, envp*/); + if(rete == -1) { + fail(4); + } else { + // unreachable code, exec never returns if it succeeds + assert(0); + } + } else { + pid = forkRet; + + recordChildCreated(pid, this); + + // close our copy of the write side of the error reporting pipe + // so the read will immediately give eof when the fork closes it too + ErrnoEnforce!close(errorReportPipes[1]); + + int[2] msg; + // this will block to wait for it to actually either start up or fail to exec (which should be near instant) + auto val = read(errorReportPipes[0], msg.ptr, msg.sizeof); + + if(val == -1) + throw new ErrnoApiException("read error report", errno); + + if(val == msg.sizeof) { + // error happened + // FIXME: keep the step part of the error report too + throw new ErrnoApiException("exec", msg[1]); + } else if(val == 0) { + // pipe closed, meaning exec succeeded + } else { + assert(0); // never supposed to happen + } + + // set the ones we keep to close upon future execs + // FIXME should i set NOBLOCK at this time too? prolly should + setCloExec(stdinPipes[1]); + setCloExec(stdoutPipes[0]); + setCloExec(stderrPipes[0]); + + // and close the others + ErrnoEnforce!close(stdinPipes[0]); + ErrnoEnforce!close(stdoutPipes[1]); + ErrnoEnforce!close(stderrPipes[1]); + + ErrnoEnforce!close(errorReportPipes[0]); + + // and now register the ones we need to read with the event loop so it can call the callbacks + // also need to listen to SIGCHLD to queue up the terminated callback. FIXME + + stdoutUnregisterToken = getThisThreadEventLoop().addCallbackOnFdReadable(stdoutFd, new CallbackHelper(&stdoutReadable)); + stderrUnregisterToken = getThisThreadEventLoop().addCallbackOnFdReadable(stderrFd, new CallbackHelper(&stderrReadable)); + } + } + } + + private version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + + int stdinFd = -1; + int stdoutFd = -1; + int stderrFd = -1; + + ICoreEventLoop.UnregisterToken stdoutUnregisterToken; + ICoreEventLoop.UnregisterToken stderrUnregisterToken; + + pid_t pid = -1; + + public void delegate() beforeExec; + + FilePath program; + string[] args; + + void stdoutReadable() { + if(stdoutReadBuffer is null) + stdoutReadBuffer = new ubyte[](stdoutBufferSize); + auto ret = read(stdoutFd, stdoutReadBuffer.ptr, stdoutReadBuffer.length); + if(ret == -1) + throw new ErrnoApiException("read", errno); + if(onStdoutAvailable) { + onStdoutAvailable(stdoutReadBuffer[0 .. ret]); + } + + if(ret == 0) { + stdoutUnregisterToken.unregister(); + + close(stdoutFd); + stdoutFd = -1; + } + } + + void stderrReadable() { + if(stderrReadBuffer is null) + stderrReadBuffer = new ubyte[](stderrBufferSize); + auto ret = read(stderrFd, stderrReadBuffer.ptr, stderrReadBuffer.length); + if(ret == -1) + throw new ErrnoApiException("read", errno); + if(onStderrAvailable) { + onStderrAvailable(stderrReadBuffer[0 .. ret]); + } + + if(ret == 0) { + stderrUnregisterToken.unregister(); + + close(stderrFd); + stderrFd = -1; + } + } + } + + private ubyte[] stdoutReadBuffer; + private ubyte[] stderrReadBuffer; + + void waitForCompletion() { + getThisThreadEventLoop().run(&this.isComplete); + } + + bool isComplete() { + return completed; + } + + bool completed; + int status = int.min; + + /++ + If blocking, it will block the current task until the write succeeds. + + Write `null` as data to close the pipe. Once the pipe is closed, you must not try to write to it again. + +/ + void writeToStdin(in void[] data) { + version(Posix) { + if(data is null) { + close(stdinFd); + stdinFd = -1; + } else { + // FIXME: check the return value again and queue async writes + auto ret = write(stdinFd, data.ptr, data.length); + if(ret == -1) + throw new ErrnoApiException("write", errno); + } + } + + } + + void delegate(ubyte[] got) onStdoutAvailable; + void delegate(ubyte[] got) onStderrAvailable; + void delegate(int code) onTermination; + + // pty? +} + +// FIXME: comment this out +/+ +unittest { + auto proc = new ExternalProcess(FilePath("/bin/cat"), ["/bin/cat"]); + + getThisThreadEventLoop(); // initialize it + + int c = 0; + proc.onStdoutAvailable = delegate(ubyte[] got) { + if(c == 0) + assert(cast(string) got == "hello!"); + else + assert(got.length == 0); + // import std.stdio; writeln(got); + c++; + }; + + proc.start(); + + assert(proc.pid != -1); + + + import std.stdio; + Thread[4] pool; + + bool shouldExit; + + static int received; + + proc.writeToStdin("hello!"); + proc.writeToStdin(null); // closes the pipe + + proc.waitForCompletion(); + + assert(proc.status == 0); + + assert(c == 2); + + // writeln("here"); +} ++/ + +// to test the thundering herd on signal handling +version(none) +unittest { + Thread[4] pool; + foreach(ref thread; pool) { + thread = new class Thread { + this() { + super({ + int count; + getThisThreadEventLoop().run(() { + if(count > 4) return true; + count++; + return false; + }); + }); + } + }; + thread.start(); + } + foreach(ref thread; pool) { + thread.join(); + } +} + +/+ + ================ + LOGGER FRAMEWORK + ================ ++/ +/++ + The arsd.core logger works differently than many in that it works as a ring buffer of objects that are consumed (or missed; buffer overruns are possible) by a different thread instead of as strings written to some file. + + A library (or an application) defines a log source. They write to this source. + + Applications then define log sinks, zero or more, which reads from various sources and does something with them. + + Log calls, in this sense, are quite similar to asynchronous events that can be subscribed to by event handlers. The difference is events are generally not dropped - they might coalesce but are usually not just plain dropped in a buffer overrun - whereas logs can be. If the log consumer can't keep up, the details are just lost. The log producer will not wait for the consumer to catch up. + + + An application can also set a default subscriber which applies to all log objects throughout. + + All log message objects must be capable of being converted to strings and to json. + + Ad-hoc messages can be done with interpolated sequences. + + Messages automatically get a timestamp. They can also have file/line and maybe even a call stack. + + Examples: + --- + mixin LoggerOf!X mylogger; + + mylogger.log(i"$this heartbeat"); // creates an ad-hoc log message + --- + + History: + Added May 27, 2024 ++/ +mixin template LoggerOf(T) { + void log(LogLevel l, T message) { + + } +} + +private static template WillFitInGeis(Args...) { + static int lengthRequired() { + int place; + foreach(arg; Args) { + static if(is(arg == InterpolatedLiteral!str, string str)) { + if(place & 1) // can't put string in the data slot + place++; + place++; + } else static if(is(arg == InterpolationHeader) || is(arg == InterpolationFooter) || is(arg == InterpolatedExpression!code, string code)) { + // no storage required + } else { + if((place & 1) == 0) // can't put data in the string slot + place++; + place++; + } + } + + if(place & 1) + place++; + return place / 2; + } + + enum WillFitInGeis = lengthRequired() <= GenericEmbeddableInterpolatedSequence.seq.length; +} + + +/+ + For making an array of istrings basically; it moves their CT magic to RT dynamic type. ++/ +struct GenericEmbeddableInterpolatedSequence { + static struct Element { + string str; // these are pointers to string literals every time + LimitedVariant lv; + } + + Element[8] seq; + + this(Args...)(InterpolationHeader, Args args, InterpolationFooter) { + int place; + bool stringUsedInPlace; + bool overflowed; + + static assert(WillFitInGeis!(Args), "Your interpolated elements will not fit in the generic buffer."); + + foreach(arg; args) { + static if(is(typeof(arg) == InterpolatedLiteral!str, string str)) { + if(stringUsedInPlace) { + place++; + stringUsedInPlace = false; + } + + if(place == seq.length) { + overflowed = true; + break; + } + seq[place].str = str; + stringUsedInPlace = true; + } else static if(is(typeof(arg) == InterpolationHeader) || is(typeof(arg) == InterpolationFooter)) { + static assert(0, "Cannot embed interpolated sequences"); + } else static if(is(typeof(arg) == InterpolatedExpression!code, string code)) { + // irrelevant + } else { + if(place == seq.length) { + overflowed = true; + break; + } + seq[place].lv = LimitedVariant(arg); + place++; + stringUsedInPlace = false; + } + } + } + + string toString() { + string s; + foreach(item; seq) { + if(item.str !is null) + s ~= item.str; + if(!item.lv.containsNull()) + s ~= item.lv.toString(); + } + return s; + } +} + +private struct LoggedElement(T) { + LogLevel level; // ? + MonoTime timestamp; + void*[16] stack; // ? + string originComponent; + string originFile; + size_t originLine; + + T message; +} + +private class TypeErasedLogger { + ubyte[] buffer; + + void*[] messagePointers; + size_t position; +} + + + + +/+ + ================= + STDIO REPLACEMENT + ================= ++/ + +private void appendToBuffer(ref char[] buffer, ref int pos, scope const(char)[] what) { + auto required = pos + what.length; + if(buffer.length < required) + buffer.length = required; + buffer[pos .. pos + what.length] = what[]; + pos += what.length; +} + +private void appendToBuffer(ref char[] buffer, ref int pos, long what) { + if(buffer.length < pos + 16) + buffer.length = pos + 16; + auto sliced = intToString(what, buffer[pos .. $]); + pos += sliced.length; +} + +/++ + A `writeln` that actually works, at least for some basic types. + + It works correctly on Windows, using the correct functions to write unicode to the console. even allocating a console if needed. If the output has been redirected to a file or pipe, it writes UTF-8. + + This always does text. See also WritableStream and WritableTextStream when they are implemented. ++/ +void writeln(T...)(T t) { + char[256] bufferBacking; + char[] buffer = bufferBacking[]; + int pos; + + foreach(arg; t) { + static if(is(typeof(arg) : const char[])) { + appendToBuffer(buffer, pos, arg); + } else static if(is(typeof(arg) : stringz)) { + appendToBuffer(buffer, pos, arg.borrow); + } else static if(is(typeof(arg) : long)) { + appendToBuffer(buffer, pos, arg); + } else static if(is(typeof(arg.toString()) : const char[])) { + appendToBuffer(buffer, pos, arg.toString()); + } else { + appendToBuffer(buffer, pos, "<" ~ typeof(arg).stringof ~ ">"); + } + } + + appendToBuffer(buffer, pos, "\n"); + + actuallyWriteToStdout(buffer[0 .. pos]); +} + +private void actuallyWriteToStdout(scope char[] buffer) @trusted { + + version(UseStdioWriteln) + { + import std.stdio; + writeln(buffer); + } + else version(Windows) { + import core.sys.windows.wincon; + + auto hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + if(hStdOut == null || hStdOut == INVALID_HANDLE_VALUE) { + AllocConsole(); + hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + } + + if(GetFileType(hStdOut) == FILE_TYPE_CHAR) { + wchar[256] wbuffer; + auto toWrite = makeWindowsString(buffer, wbuffer, WindowsStringConversionFlags.convertNewLines); + + DWORD written; + WriteConsoleW(hStdOut, toWrite.ptr, cast(DWORD) toWrite.length, &written, null); + } else { + DWORD written; + WriteFile(hStdOut, buffer.ptr, cast(DWORD) buffer.length, &written, null); + } + } else { + import unix = core.sys.posix.unistd; + unix.write(1, buffer.ptr, buffer.length); + } +} + +/+ + +STDIO + + /++ + Please note using this will create a compile-time dependency on [arsd.terminal] + + + +so my writeln replacement: + +1) if the std output handle is null, alloc one +2) if it is a character device, write out the proper Unicode text. +3) otherwise write out UTF-8.... maybe with a BOM but maybe not. it is tricky to know what the other end of a pipe expects... +[8:15 AM] +im actually tempted to make the write binary to stdout functions throw an exception if it is a character console / interactive terminal instead of letting you spam it right out +[8:16 AM] +of course you can still cheat by casting binary data to string and using the write string function (and this might be appropriate sometimes) but there kinda is a legit difference between a text output and a binary output device + +Stdout can represent either + + +/ + void writeln(){} { + + } + + stderr? + + /++ + Please note using this will create a compile-time dependency on [arsd.terminal] + + It can be called from a task. + + It works correctly on Windows and is user friendly on Linux (using arsd.terminal.getline) + while also working if stdin has been redirected (where arsd.terminal itself would throw) + + +so say you run a program on an interactive terminal. the program tries to open the stdin binary stream + +instead of throwing, the prompt could change to indicate the binary data is expected and you can feed it in either by typing it up,,,, or running some command like maybe defaultModel!Type(defaultModel("identifier")) + + + + + + + + + + +so while i laid there sleep deprived i did think a lil more on some uda stuff. it isn't especially novel but a combination of a few other techniques + +you might be like + +struct MyUdas { + DbName name; + DbIgnore ignore; +} + +elsewhere + +foreach(alias; allMembers) { + auto udas = getUdas!(MyUdas, __traits(getAttributes, alias))(MyUdas(DbName(__traits(identifier, alias)))); +} + + +so you pass the expected type and the attributes as the template params, then the runtime params are the default values for the given types + +so what the thing does essentially is just sets the values of the given thing to the udas based on type then returns the modified instance + +so the end result is you keep the last ones. it wouldn't report errors if multiple things added but it p simple to understand, simple to document (even though the default values are not in the struct itself, you can put ddocs in them), and uses the tricks to minimize generated code size ++/ + ++/ + +package(arsd) version(Windows) extern(Windows) { + BOOL CancelIoEx(HANDLE, LPOVERLAPPED); + + struct WSABUF { + ULONG len; + ubyte* buf; + } + alias LPWSABUF = WSABUF*; + + // https://learn.microsoft.com/en-us/windows/win32/api/winsock2/ns-winsock2-wsaoverlapped + // "The WSAOVERLAPPED structure is compatible with the Windows OVERLAPPED structure." + // so ima lie here in the bindings. + + int WSASend(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); + int WSASendTo(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, const sockaddr*, int, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); + + int WSARecv(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); + int WSARecvFrom(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, sockaddr*, LPINT, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); +} + +package(arsd) version(OSXCocoa) { + +/* Copy/paste chunk from Jacob Carlborg { */ +// from https://raw.githubusercontent.com/jacob-carlborg/druntime/550edd0a64f0eb2c4f35d3ec3d88e26b40ac779e/src/core/stdc/clang_block.d +// with comments stripped (see docs in the original link), code reformatted, and some names changed to avoid potential conflicts + +import core.stdc.config; +struct ObjCBlock(R = void, Params...) { +private: + alias extern(C) R function(ObjCBlock*, Params) Invoke; + + void* isa; + int flags; + int reserved = 0; + Invoke invoke; + Descriptor* descriptor; + + // Imported variables go here + R delegate(Params) dg; + + this(void* isa, int flags, Invoke invoke, R delegate(Params) dg) { + this.isa = isa; + this.flags = flags; + this.invoke = invoke; + this.dg = dg; + this.descriptor = &.objcblock_descriptor; + } +} +ObjCBlock!(R, Params) block(R, Params...)(R delegate(Params) dg) { + static if (Params.length == 0) + enum flags = 0x50000000; + else + enum flags = 0x40000000; + + return ObjCBlock!(R, Params)(&_NSConcreteStackBlock, flags, &objcblock_invoke!(R, Params), dg); +} + +private struct Descriptor { + c_ulong reserved; + c_ulong size; + const(char)* signature; +} +private extern(C) extern __gshared void*[32] _NSConcreteStackBlock; +private __gshared auto objcblock_descriptor = Descriptor(0, ObjCBlock!().sizeof); +private extern(C) R objcblock_invoke(R, Args...)(ObjCBlock!(R, Args)* block, Args args) { + return block.dg(args); +} + + +/* End copy/paste chunk from Jacob Carlborg } */ + + +/+ +To let Cocoa know that you intend to use multiple threads, all you have to do is spawn a single thread using the NSThread class and let that thread immediately exit. Your thread entry point need not do anything. Just the act of spawning a thread using NSThread is enough to ensure that the locks needed by the Cocoa frameworks are put in place. + +If you are not sure if Cocoa thinks your application is multithreaded or not, you can use the isMultiThreaded method of NSThread to check. ++/ + + + struct DeifiedNSString { + char[16] sso; + const(char)[] str; + + this(NSString s) { + auto len = s.length; + if(len <= sso.length / 4) + str = sso[]; + else + str = new char[](len * 4); + + NSUInteger count; + NSRange leftover; + auto ret = s.getBytes(cast(char*) str.ptr, str.length, &count, NSStringEncoding.NSUTF8StringEncoding, NSStringEncodingConversionOptions.none, NSRange(0, len), &leftover); + if(ret) + str = str[0 .. count]; + else + throw new Exception("uh oh"); + } + } + + extern (Objective-C) { + import core.attribute; // : selector, optional; + + alias NSUInteger = size_t; + alias NSInteger = ptrdiff_t; + alias unichar = wchar; + struct SEL_; + alias SEL_* SEL; + // this is called plain `id` in objective C but i fear mistakes with that in D. like sure it is a type instead of a variable like most things called id but i still think it is weird. i might change my mind later. + alias void* NSid; // FIXME? the docs say this is a pointer to an instance of a class, but that is not necessary a child of NSObject + + extern class NSObject { + static NSObject alloc() @selector("alloc"); + NSObject init() @selector("init"); + + void retain() @selector("retain"); + void release() @selector("release"); + void autorelease() @selector("autorelease"); + + void performSelectorOnMainThread(SEL aSelector, NSid arg, bool waitUntilDone) @selector("performSelectorOnMainThread:withObject:waitUntilDone:"); + } + + // this is some kind of generic in objc... + extern class NSArray : NSObject { + static NSArray arrayWithObjects(NSid* objects, NSUInteger count) @selector("arrayWithObjects:count:"); + } + + extern class NSString : NSObject { + override static NSString alloc() @selector("alloc"); + override NSString init() @selector("init"); + + NSString initWithUTF8String(const scope char* str) @selector("initWithUTF8String:"); + + NSString initWithBytes( + const(ubyte)* bytes, + NSUInteger length, + NSStringEncoding encoding + ) @selector("initWithBytes:length:encoding:"); + + unichar characterAtIndex(NSUInteger index) @selector("characterAtIndex:"); + NSUInteger length() @selector("length"); + const char* UTF8String() @selector("UTF8String"); + + void getCharacters(wchar* buffer, NSRange range) @selector("getCharacters:range:"); + + bool getBytes(void* buffer, NSUInteger maxBufferCount, NSUInteger* usedBufferCount, NSStringEncoding encoding, NSStringEncodingConversionOptions options, NSRange range, NSRange* leftover) @selector("getBytes:maxLength:usedLength:encoding:options:range:remainingRange:"); + } + + struct NSRange { + NSUInteger loc; + NSUInteger len; + } + + enum NSStringEncodingConversionOptions : NSInteger { + none = 0, + NSAllowLossyEncodingConversion = 1, + NSExternalRepresentationEncodingConversion = 2 + } + + enum NSEventType { + idk + + } + + enum NSEventModifierFlags : NSUInteger { + NSEventModifierFlagCapsLock = 1 << 16, + NSEventModifierFlagShift = 1 << 17, + NSEventModifierFlagControl = 1 << 18, + NSEventModifierFlagOption = 1 << 19, // aka Alt + NSEventModifierFlagCommand = 1 << 20, // aka super + NSEventModifierFlagNumericPad = 1 << 21, + NSEventModifierFlagHelp = 1 << 22, + NSEventModifierFlagFunction = 1 << 23, + NSEventModifierFlagDeviceIndependentFlagsMask = 0xffff0000UL + } + + extern class NSEvent : NSObject { + NSEventType type() @selector("type"); + + NSPoint locationInWindow() @selector("locationInWindow"); + NSTimeInterval timestamp() @selector("timestamp"); + NSWindow window() @selector("window"); // note: nullable + NSEventModifierFlags modifierFlags() @selector("modifierFlags"); + + NSString characters() @selector("characters"); + NSString charactersIgnoringModifiers() @selector("charactersIgnoringModifiers"); + ushort keyCode() @selector("keyCode"); + ushort specialKey() @selector("specialKey"); + + static NSUInteger pressedMouseButtons() @selector("pressedMouseButtons"); + NSPoint locationInWindow() @selector("locationInWindow"); // in screen coordinates + static NSPoint mouseLocation() @selector("mouseLocation"); // in screen coordinates + NSInteger buttonNumber() @selector("buttonNumber"); + + CGFloat deltaX() @selector("deltaX"); + CGFloat deltaY() @selector("deltaY"); + CGFloat deltaZ() @selector("deltaZ"); + + bool hasPreciseScrollingDeltas() @selector("hasPreciseScrollingDeltas"); + + CGFloat scrollingDeltaX() @selector("scrollingDeltaX"); + CGFloat scrollingDeltaY() @selector("scrollingDeltaY"); + + // @property(getter=isDirectionInvertedFromDevice, readonly) BOOL directionInvertedFromDevice; + } + + extern /* final */ class NSTimer : NSObject { // the docs say don't subclass this, but making it final breaks the bridge + override static NSTimer alloc() @selector("alloc"); + override NSTimer init() @selector("init"); + + static NSTimer schedule(NSTimeInterval timeIntervalInSeconds, NSid target, SEL selector, NSid userInfo, bool repeats) @selector("scheduledTimerWithTimeInterval:target:selector:userInfo:repeats:"); + + void fire() @selector("fire"); + void invalidate() @selector("invalidate"); + + bool valid() @selector("isValid"); + // @property(copy) NSDate *fireDate; + NSTimeInterval timeInterval() @selector("timeInterval"); + NSid userInfo() @selector("userInfo"); + + NSTimeInterval tolerance() @selector("tolerance"); + NSTimeInterval tolerance(NSTimeInterval) @selector("setTolerance:"); + } + + alias NSTimeInterval = double; + + extern class NSResponder : NSObject { + NSMenu menu() @selector("menu"); + void menu(NSMenu menu) @selector("setMenu:"); + + void keyDown(NSEvent event) @selector("keyDown:"); + void keyUp(NSEvent event) @selector("keyUp:"); + + // - (void)interpretKeyEvents:(NSArray *)eventArray; + + void mouseDown(NSEvent event) @selector("mouseDown:"); + void mouseDragged(NSEvent event) @selector("mouseDragged:"); + void mouseUp(NSEvent event) @selector("mouseUp:"); + void mouseMoved(NSEvent event) @selector("mouseMoved:"); + void mouseEntered(NSEvent event) @selector("mouseEntered:"); + void mouseExited(NSEvent event) @selector("mouseExited:"); + + void rightMouseDown(NSEvent event) @selector("rightMouseDown:"); + void rightMouseDragged(NSEvent event) @selector("rightMouseDragged:"); + void rightMouseUp(NSEvent event) @selector("rightMouseUp:"); + + void otherMouseDown(NSEvent event) @selector("otherMouseDown:"); + void otherMouseDragged(NSEvent event) @selector("otherMouseDragged:"); + void otherMouseUp(NSEvent event) @selector("otherMouseUp:"); + + void scrollWheel(NSEvent event) @selector("scrollWheel:"); + + // touch events should also be here btw among others + } + + extern class NSApplication : NSResponder { + static NSApplication shared_() @selector("sharedApplication"); + + NSApplicationDelegate delegate_() @selector("delegate"); + void delegate_(NSApplicationDelegate) @selector("setDelegate:"); + + bool setActivationPolicy(NSApplicationActivationPolicy activationPolicy) @selector("setActivationPolicy:"); + + void activateIgnoringOtherApps(bool flag) @selector("activateIgnoringOtherApps:"); + + @property NSMenu mainMenu() @selector("mainMenu"); + @property NSMenu mainMenu(NSMenu) @selector("setMainMenu:"); + + void run() @selector("run"); + + void terminate(void*) @selector("terminate:"); + } + + extern interface NSApplicationDelegate { + void applicationWillFinishLaunching(NSNotification notification) @selector("applicationWillFinishLaunching:"); + void applicationDidFinishLaunching(NSNotification notification) @selector("applicationDidFinishLaunching:"); + bool applicationShouldTerminateAfterLastWindowClosed(NSNotification notification) @selector("applicationShouldTerminateAfterLastWindowClosed:"); + } + + extern class NSNotification : NSObject { + @property NSid object() @selector("object"); + } + + enum NSApplicationActivationPolicy : ptrdiff_t { + /* The application is an ordinary app that appears in the Dock and may have a user interface. This is the default for bundled apps, unless overridden in the Info.plist. */ + regular, + + /* The application does not appear in the Dock and does not have a menu bar, but it may be activated programmatically or by clicking on one of its windows. This corresponds to LSUIElement=1 in the Info.plist. */ + accessory, + + /* The application does not appear in the Dock and may not create windows or be activated. This corresponds to LSBackgroundOnly=1 in the Info.plist. This is also the default for unbundled executables that do not have Info.plists. */ + prohibited + } + + extern class NSGraphicsContext : NSObject { + static NSGraphicsContext currentContext() @selector("currentContext"); + NSGraphicsContext graphicsPort() @selector("graphicsPort"); + } + + extern class NSMenu : NSObject { + override static NSMenu alloc() @selector("alloc"); + + override NSMenu init() @selector("init"); + NSMenu init(NSString title) @selector("initWithTitle:"); + + void setSubmenu(NSMenu menu, NSMenuItem item) @selector("setSubmenu:forItem:"); + void addItem(NSMenuItem newItem) @selector("addItem:"); + + NSMenuItem addItem( + NSString title, + SEL selector, + NSString charCode + ) @selector("addItemWithTitle:action:keyEquivalent:"); + } + + extern class NSMenuItem : NSObject { + override static NSMenuItem alloc() @selector("alloc"); + override NSMenuItem init() @selector("init"); + + NSMenuItem init( + NSString title, + SEL selector, + NSString charCode + ) @selector("initWithTitle:action:keyEquivalent:"); + + void enabled(bool) @selector("setEnabled:"); + + NSResponder target(NSResponder) @selector("setTarget:"); + } + + enum NSWindowStyleMask : size_t { + borderless = 0, + titled = 1 << 0, + closable = 1 << 1, + miniaturizable = 1 << 2, + resizable = 1 << 3, + + /* Specifies a window with textured background. Textured windows generally don't draw a top border line under the titlebar/toolbar. To get that line, use the NSUnifiedTitleAndToolbarWindowMask mask. + */ + texturedBackground = 1 << 8, + + /* Specifies a window whose titlebar and toolbar have a unified look - that is, a continuous background. Under the titlebar and toolbar a horizontal separator line will appear. + */ + unifiedTitleAndToolbar = 1 << 12, + + /* When set, the window will appear full screen. This mask is automatically toggled when toggleFullScreen: is called. + */ + fullScreen = 1 << 14, + + /* If set, the contentView will consume the full size of the window; it can be combined with other window style masks, but is only respected for windows with a titlebar. + Utilizing this mask opts-in to layer-backing. Utilize the contentLayoutRect or auto-layout contentLayoutGuide to layout views underneath the titlebar/toolbar area. + */ + fullSizeContentView = 1 << 15, + + /* The following are only applicable for NSPanel (or a subclass thereof) + */ + utilityWindow = 1 << 4, + docModalWindow = 1 << 6, + nonactivatingPanel = 1 << 7, // Specifies that a panel that does not activate the owning application + hUDWindow = 1 << 13 // Specifies a heads up display panel + } + + extern class NSWindow : NSObject { + override static NSWindow alloc() @selector("alloc"); + + override NSWindow init() @selector("init"); + + NSWindow initWithContentRect( + NSRect contentRect, + NSWindowStyleMask style, + NSBackingStoreType bufferingType, + bool flag + ) @selector("initWithContentRect:styleMask:backing:defer:"); + + void makeKeyAndOrderFront(NSid sender) @selector("makeKeyAndOrderFront:"); + NSView contentView() @selector("contentView"); + void contentView(NSView view) @selector("setContentView:"); + void orderFrontRegardless() @selector("orderFrontRegardless"); + void center() @selector("center"); + + NSRect frame() @selector("frame"); + + NSRect contentRectForFrameRect(NSRect frameRect) @selector("contentRectForFrameRect:"); + + NSString title() @selector("title"); + void title(NSString value) @selector("setTitle:"); + + void close() @selector("close"); + + NSWindowDelegate delegate_() @selector("delegate"); + void delegate_(NSWindowDelegate) @selector("setDelegate:"); + + void setBackgroundColor(NSColor color) @selector("setBackgroundColor:"); + } + + extern interface NSWindowDelegate { + @optional: + void windowDidResize(NSNotification notification) @selector("windowDidResize:"); + + NSSize windowWillResize(NSWindow sender, NSSize frameSize) @selector("windowWillResize:toSize:"); + + void windowWillClose(NSNotification notification) @selector("windowWillClose:"); + } + + extern class NSView : NSResponder { + override NSView init() @selector("init"); + NSView initWithFrame(NSRect frameRect) @selector("initWithFrame:"); + + void addSubview(NSView view) @selector("addSubview:"); + + bool wantsLayer() @selector("wantsLayer"); + void wantsLayer(bool value) @selector("setWantsLayer:"); + + CALayer layer() @selector("layer"); + void uiDelegate(NSObject) @selector("setUIDelegate:"); + + void drawRect(NSRect rect) @selector("drawRect:"); + bool isFlipped() @selector("isFlipped"); + bool acceptsFirstResponder() @selector("acceptsFirstResponder"); + bool setNeedsDisplay(bool) @selector("setNeedsDisplay:"); + + // DO NOT USE: https://issues.dlang.org/show_bug.cgi?id=19017 + // an asm { pop RAX; } after getting the struct can kinda hack around this but still + @property NSRect frame() @selector("frame"); + @property NSRect frame(NSRect rect) @selector("setFrame:"); + + void setFrameSize(NSSize newSize) @selector("setFrameSize:"); + void setFrameOrigin(NSPoint newOrigin) @selector("setFrameOrigin:"); + + void addSubview(NSView what) @selector("addSubview:"); + void removeFromSuperview() @selector("removeFromSuperview"); + } + + extern class NSFont : NSObject { + void set() @selector("set"); // sets it into the current graphics context + void setInContext(NSGraphicsContext context) @selector("setInContext:"); + + static NSFont fontWithName(NSString fontName, CGFloat fontSize) @selector("fontWithName:size:"); + // fontWithDescriptor too + // fontWithName and matrix too + static NSFont systemFontOfSize(CGFloat fontSize) @selector("systemFontOfSize:"); + // among others + + @property CGFloat pointSize() @selector("pointSize"); + @property bool isFixedPitch() @selector("isFixedPitch"); + // fontDescriptor + @property NSString displayName() @selector("displayName"); + + @property CGFloat ascender() @selector("ascender"); + @property CGFloat descender() @selector("descender"); // note it is negative + @property CGFloat capHeight() @selector("capHeight"); + @property CGFloat leading() @selector("leading"); + @property CGFloat xHeight() @selector("xHeight"); + // among many more + } + + extern class NSColor : NSObject { + override static NSColor alloc() @selector("alloc"); + static NSColor redColor() @selector("redColor"); + static NSColor whiteColor() @selector("whiteColor"); + + CGColorRef CGColor() @selector("CGColor"); + } + + extern class CALayer : NSObject { + CGFloat borderWidth() @selector("borderWidth"); + void borderWidth(CGFloat value) @selector("setBorderWidth:"); + + CGColorRef borderColor() @selector("borderColor"); + void borderColor(CGColorRef) @selector("setBorderColor:"); + } + + + extern class NSViewController : NSObject { + NSView view() @selector("view"); + void view(NSView view) @selector("setView:"); + } + + enum NSBackingStoreType : size_t { + retained = 0, + nonretained = 1, + buffered = 2 + } + + enum NSStringEncoding : NSUInteger { + NSASCIIStringEncoding = 1, /* 0..127 only */ + NSUTF8StringEncoding = 4, + NSUnicodeStringEncoding = 10, + + NSUTF16StringEncoding = NSUnicodeStringEncoding, + NSUTF16BigEndianStringEncoding = 0x90000100, + NSUTF16LittleEndianStringEncoding = 0x94000100, + NSUTF32StringEncoding = 0x8c000100, + NSUTF32BigEndianStringEncoding = 0x98000100, + NSUTF32LittleEndianStringEncoding = 0x9c000100 + } + + + struct CGColor; + alias CGColorRef = CGColor*; + + // note on the watch os it is float, not double + alias CGFloat = double; + + struct NSPoint { + CGFloat x; + CGFloat y; + } + + struct NSSize { + CGFloat width; + CGFloat height; + } + + struct NSRect { + NSPoint origin; + NSSize size; + } + + alias NSPoint CGPoint; + alias NSSize CGSize; + alias NSRect CGRect; + + pragma(inline, true) NSPoint NSMakePoint(CGFloat x, CGFloat y) { + NSPoint p; + p.x = x; + p.y = y; + return p; + } + + pragma(inline, true) NSSize NSMakeSize(CGFloat w, CGFloat h) { + NSSize s; + s.width = w; + s.height = h; + return s; + } + + pragma(inline, true) NSRect NSMakeRect(CGFloat x, CGFloat y, CGFloat w, CGFloat h) { + NSRect r; + r.origin.x = x; + r.origin.y = y; + r.size.width = w; + r.size.height = h; + return r; + } + + + } + + // helper raii refcount object + static if(UseCocoa) + struct MacString { + union { + // must be wrapped cuz of bug in dmd + // referencing an init symbol when it should + // just be null. but the union makes it work + NSString s; + } + + // FIXME: if a string literal it would be kinda nice to use + // the other function. but meh + + this(scope const char[] str) { + this.s = NSString.alloc.initWithBytes( + cast(const(ubyte)*) str.ptr, + str.length, + NSStringEncoding.NSUTF8StringEncoding + ); + } + + NSString borrow() { + return s; + } + + this(this) { + if(s !is null) + s.retain(); + } + + ~this() { + if(s !is null) { + s.release(); + s = null; + } + } + } + + extern(C) void NSLog(NSString, ...); + extern(C) SEL sel_registerName(const(char)* str); + + extern (Objective-C) __gshared NSApplication NSApp_; + + NSApplication NSApp() { + if(NSApp_ is null) + NSApp_ = NSApplication.shared_; + return NSApp_; + } + + // hacks to work around compiler bug + extern(C) __gshared void* _D4arsd4core17NSGraphicsContext7__ClassZ = null; + extern(C) __gshared void* _D4arsd4core6NSView7__ClassZ = null; + extern(C) __gshared void* _D4arsd4core8NSWindow7__ClassZ = null; +} diff --git a/arsd/dom.d b/arsd/dom.d new file mode 100644 index 0000000..fb3dabb --- /dev/null +++ b/arsd/dom.d @@ -0,0 +1,8778 @@ +// FIXME: xml namespace support??? +// FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML +// FIXME: parentElement is parentNode that skips DocumentFragment etc but will be hard to work in with my compatibility... + +// FIXME: the scriptable list is quite arbitrary + + +// xml entity references?! + +/++ + This is an html DOM implementation, started with cloning + what the browser offers in Javascript, but going well beyond + it in convenience. + + If you can do it in Javascript, you can probably do it with + this module, and much more. + + --- + import arsd.dom; + + void main() { + auto document = new Document("

    paragraph

    "); + writeln(document.querySelector("p")); + document.root.innerHTML = "

    hey

    "; + writeln(document); + } + --- + + BTW: this file optionally depends on `arsd.characterencodings`, to + help it correctly read files from the internet. You should be able to + get characterencodings.d from the same place you got this file. + + If you want it to stand alone, just always use the `Document.parseUtf8` + function or the constructor that takes a string. + + Symbol_groups: + + core_functionality = + + These members provide core functionality. The members on these classes + will provide most your direct interaction. + + bonus_functionality = + + These provide additional functionality for special use cases. + + implementations = + + These provide implementations of other functionality. ++/ +module arsd.dom; + +static import arsd.core; +import arsd.core : encodeUriComponent, decodeUriComponent; + +// FIXME: support the css standard namespace thing in the selectors too + +version(with_arsd_jsvar) + import arsd.jsvar; +else { + enum scriptable = "arsd_jsvar_compatible"; +} + +// this is only meant to be used at compile time, as a filter for opDispatch +// lists the attributes we want to allow without the use of .attr +bool isConvenientAttribute(string name) { + static immutable list = [ + "name", "id", "href", "value", + "checked", "selected", "type", + "src", "content", "pattern", + "placeholder", "required", "alt", + "rel", + "method", "action", "enctype" + ]; + foreach(l; list) + if(name == l) return true; + return false; +} + + +// FIXME: something like
      spam
        with no closing
      should read the second tag as the closer in garbage mode +// FIXME: failing to close a paragraph sometimes messes things up too + +// FIXME: it would be kinda cool to have some support for internal DTDs +// and maybe XPath as well, to some extent +/* + we could do + meh this sux + + auto xpath = XPath(element); + + // get the first p + xpath.p[0].a["href"] +*/ + + +/++ + The main document interface, including a html or xml parser. + + There's three main ways to create a Document: + + If you want to parse something and inspect the tags, you can use the [this|constructor]: + --- + // create and parse some HTML in one call + auto document = new Document(""); + + // or some XML + auto document = new Document("", true, true); // strict mode enabled + + // or better yet: + auto document = new XmlDocument(""); // specialized subclass + --- + + If you want to download something and parse it in one call, the [fromUrl] static function can help: + --- + auto document = Document.fromUrl("http://dlang.org/"); + --- + (note that this requires my [arsd.characterencodings] and [arsd.http2] libraries) + + And, if you need to inspect things like `<%= foo %>` tags and comments, you can add them to the dom like this, with the [enableAddingSpecialTagsToDom] + and [parseUtf8] or [parseGarbage] functions: + --- + auto document = new Document(); + document.enableAddingSpecialTagsToDom(); + document.parseUtf8("", true, true); // changes the trues to false to switch from xml to html mode + --- + + You can also modify things like [selfClosedElements] and [rawSourceElements] before calling the `parse` family of functions to do further advanced tasks. + + However you parse it, it will put a few things into special variables. + + [root] contains the root document. + [prolog] contains the instructions before the root (like ``). To keep the original things, you will need to [enableAddingSpecialTagsToDom] first, otherwise the library will return generic strings in there. [piecesBeforeRoot] will have other parsed instructions, if [enableAddingSpecialTagsToDom] is called. + [piecesAfterRoot] will contain any xml-looking data after the root tag is closed. + + Most often though, you will not need to look at any of that data, since `Document` itself has methods like [querySelector], [appendChild], and more which will forward to the root [Element] for you. ++/ +/// Group: core_functionality +class Document : FileResource, DomParent { + inout(Document) asDocument() inout { return this; } + inout(Element) asElement() inout { return null; } + + void processNodeWhileParsing(Element parent, Element child) { + parent.appendChild(child); + } + + /++ + Convenience method for web scraping. Requires [arsd.http2] to be + included in the build as well as [arsd.characterencodings]. + + This will download the file from the given url and create a document + off it, using a strict constructor or a [parseGarbage], depending on + the value of `strictMode`. + +/ + static Document fromUrl()(string url, bool strictMode = false) { + import arsd.http2; + auto client = new HttpClient(); + + auto req = client.navigateTo(Uri(url), HttpVerb.GET); + auto res = req.waitForCompletion(); + + auto document = new Document(); + if(strictMode) { + document.parse(cast(string) res.content, true, true, res.contentTypeCharset); + } else { + document.parseGarbage(cast(string) res.content); + } + + return document; + } + + /++ + Creates a document with the given source data. If you want HTML behavior, use `caseSensitive` and `struct` set to `false`. For XML mode, set them to `true`. + + Please note that anything after the root element will be found in [piecesAfterRoot]. Comments, processing instructions, and other special tags will be stripped out b default. You can customize this by using the zero-argument constructor and setting callbacks on the [parseSawComment], [parseSawBangInstruction], [parseSawAspCode], [parseSawPhpCode], and [parseSawQuestionInstruction] members, then calling one of the [parseUtf8], [parseGarbage], or [parse] functions. Calling the convenience method, [enableAddingSpecialTagsToDom], will enable all those things at once. + + See_Also: + [parseGarbage] + [parseUtf8] + [parseUrl] + +/ + this(string data, bool caseSensitive = false, bool strict = false) { + parseUtf8(data, caseSensitive, strict); + } + + /** + Creates an empty document. It has *nothing* in it at all, ready. + */ + this() { + + } + + /++ + This is just something I'm toying with. Right now, you use opIndex to put in css selectors. + It returns a struct that forwards calls to all elements it holds, and returns itself so you + can chain it. + + Example: document["p"].innerText("hello").addClass("modified"); + + Equivalent to: foreach(e; document.getElementsBySelector("p")) { e.innerText("hello"); e.addClas("modified"); } + + Note: always use function calls (not property syntax) and don't use toString in there for best results. + + You can also do things like: document["p"]["b"] though tbh I'm not sure why since the selector string can do all that anyway. Maybe + you could put in some kind of custom filter function tho. + +/ + ElementCollection opIndex(string selector) { + auto e = ElementCollection(this.root); + return e[selector]; + } + + string _contentType = "text/html; charset=utf-8"; + + /// If you're using this for some other kind of XML, you can + /// set the content type here. + /// + /// Note: this has no impact on the function of this class. + /// It is only used if the document is sent via a protocol like HTTP. + /// + /// This may be called by parse() if it recognizes the data. Otherwise, + /// if you don't set it, it assumes text/html; charset=utf-8. + @property string contentType(string mimeType) { + _contentType = mimeType; + return _contentType; + } + + /// implementing the FileResource interface, useful for sending via + /// http automatically. + @property string filename() const { return null; } + + /// implementing the FileResource interface, useful for sending via + /// http automatically. + override @property string contentType() const { + return _contentType; + } + + /// implementing the FileResource interface; it calls toString. + override immutable(ubyte)[] getData() const { + return cast(immutable(ubyte)[]) this.toString(); + } + + + /* + /// Concatenates any consecutive text nodes + void normalize() { + + } + */ + + /// This will set delegates for parseSaw* (note: this overwrites anything else you set, and you setting subsequently will overwrite this) that add those things to the dom tree when it sees them. + /// Call this before calling parse(). + + /++ + Adds objects to the dom representing things normally stripped out during the default parse, like comments, ``, `<% code%>`, and `` all at once. + + Note this will also preserve the prolog and doctype from the original file, if there was one. + + See_Also: + [parseSawComment] + [parseSawAspCode] + [parseSawPhpCode] + [parseSawQuestionInstruction] + [parseSawBangInstruction] + +/ + void enableAddingSpecialTagsToDom() { + parseSawComment = (string) => true; + parseSawAspCode = (string) => true; + parseSawPhpCode = (string) => true; + parseSawQuestionInstruction = (string) => true; + parseSawBangInstruction = (string) => true; + } + + /// If the parser sees a html comment, it will call this callback + /// will call parseSawComment(" comment ") + /// Return true if you want the node appended to the document. It will be in a [HtmlComment] object. + bool delegate(string) parseSawComment; + + /// If the parser sees <% asp code... %>, it will call this callback. + /// It will be passed "% asp code... %" or "%= asp code .. %" + /// Return true if you want the node appended to the document. It will be in an [AspCode] object. + bool delegate(string) parseSawAspCode; + + /// If the parser sees , it will call this callback. + /// It will be passed "?php php code... ?" or "?= asp code .. ?" + /// Note: dom.d cannot identify the other php short format. + /// Return true if you want the node appended to the document. It will be in a [PhpCode] object. + bool delegate(string) parseSawPhpCode; + + /// if it sees a that is not php or asp + /// it calls this function with the contents. + /// calls parseSawQuestionInstruction("?SOMETHING foo") + /// Unlike the php/asp ones, this ends on the first > it sees, without requiring ?>. + /// Return true if you want the node appended to the document. It will be in a [QuestionInstruction] object. + bool delegate(string) parseSawQuestionInstruction; + + /// if it sees a calls parseSawBangInstruction("SOMETHING foo") + /// Return true if you want the node appended to the document. It will be in a [BangInstruction] object. + bool delegate(string) parseSawBangInstruction; + + /// Given the kind of garbage you find on the Internet, try to make sense of it. + /// Equivalent to document.parse(data, false, false, null); + /// (Case-insensitive, non-strict, determine character encoding from the data.) + + /// NOTE: this makes no attempt at added security, but it will try to recover from anything instead of throwing. + /// + /// It is a template so it lazily imports characterencodings. + void parseGarbage()(string data) { + parse(data, false, false, null); + } + + /// Parses well-formed UTF-8, case-sensitive, XML or XHTML + /// Will throw exceptions on things like unclosed tags. + void parseStrict(string data, bool pureXmlMode = false) { + parseStream(toUtf8Stream(data), true, true, pureXmlMode); + } + + /// Parses well-formed UTF-8 in loose mode (by default). Tries to correct + /// tag soup, but does NOT try to correct bad character encodings. + /// + /// They will still throw an exception. + void parseUtf8(string data, bool caseSensitive = false, bool strict = false) { + parseStream(toUtf8Stream(data), caseSensitive, strict); + } + + // this is a template so we get lazy import behavior + Utf8Stream handleDataEncoding()(in string rawdata, string dataEncoding, bool strict) { + import arsd.characterencodings; + // gotta determine the data encoding. If you know it, pass it in above to skip all this. + if(dataEncoding is null) { + dataEncoding = tryToDetermineEncoding(cast(const(ubyte[])) rawdata); + // it can't tell... probably a random 8 bit encoding. Let's check the document itself. + // Now, XML and HTML can both list encoding in the document, but we can't really parse + // it here without changing a lot of code until we know the encoding. So I'm going to + // do some hackish string checking. + if(dataEncoding is null) { + auto dataAsBytes = cast(immutable(ubyte)[]) rawdata; + // first, look for an XML prolog + auto idx = indexOfBytes(dataAsBytes, cast(immutable ubyte[]) "encoding=\""); + if(idx != -1) { + idx += "encoding=\"".length; + // we're probably past the prolog if it's this far in; we might be looking at + // content. Forget about it. + if(idx > 100) + idx = -1; + } + // if that fails, we're looking for Content-Type http-equiv or a meta charset (see html5).. + if(idx == -1) { + idx = indexOfBytes(dataAsBytes, cast(immutable ubyte[]) "charset="); + if(idx != -1) { + idx += "charset=".length; + if(dataAsBytes[idx] == '"') + idx++; + } + } + + // found something in either branch... + if(idx != -1) { + // read till a quote or about 12 chars, whichever comes first... + auto end = idx; + while(end < dataAsBytes.length && dataAsBytes[end] != '"' && end - idx < 12) + end++; + + dataEncoding = cast(string) dataAsBytes[idx .. end]; + } + // otherwise, we just don't know. + } + } + + if(dataEncoding is null) { + if(strict) + throw new MarkupException("I couldn't figure out the encoding of this document."); + else + // if we really don't know by here, it means we already tried UTF-8, + // looked for utf 16 and 32 byte order marks, and looked for xml or meta + // tags... let's assume it's Windows-1252, since that's probably the most + // common aside from utf that wouldn't be labeled. + + dataEncoding = "Windows 1252"; + } + + // and now, go ahead and convert it. + + string data; + + if(!strict) { + // if we're in non-strict mode, we need to check + // the document for mislabeling too; sometimes + // web documents will say they are utf-8, but aren't + // actually properly encoded. If it fails to validate, + // we'll assume it's actually Windows encoding - the most + // likely candidate for mislabeled garbage. + dataEncoding = dataEncoding.toLower(); + dataEncoding = dataEncoding.replace(" ", ""); + dataEncoding = dataEncoding.replace("-", ""); + dataEncoding = dataEncoding.replace("_", ""); + if(dataEncoding == "utf8") { + try { + validate(rawdata); + } catch(UTFException e) { + dataEncoding = "Windows 1252"; + } + } + } + + if(dataEncoding != "UTF-8") { + if(strict) + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding); + else { + try { + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding); + } catch(Exception e) { + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, "Windows 1252"); + } + } + } else + data = rawdata; + + return toUtf8Stream(data); + } + + private + Utf8Stream toUtf8Stream(in string rawdata) { + string data = rawdata; + static if(is(Utf8Stream == string)) + return data; + else + return new Utf8Stream(data); + } + + /++ + List of elements that can be assumed to be self-closed + in this document. The default for a Document are a hard-coded + list of ones appropriate for HTML. For [XmlDocument], it defaults + to empty. You can modify this after construction but before parsing. + + History: + Added February 8, 2021 (included in dub release 9.2) + + Changed from `string[]` to `immutable(string)[]` on + February 4, 2024 (dub v11.5) to plug a hole discovered + by the OpenD compiler's diagnostics. + +/ + immutable(string)[] selfClosedElements = htmlSelfClosedElements; + + /++ + List of elements that contain raw CDATA content for this + document, e.g. ` + my plaintext & stuff + `); + + // please note that if we did `document.toString()` right now, the original source - almost your same + // string you passed to parseStrict - would be spit back out. Meaning the embedded-plaintext still has its + // special text inside it. Another parser won't understand how to use this! So if you want to pass this + // document somewhere else, you need to do some transformations. + // + // This differs from cases like CDATA sections, which dom.d will automatically convert into plain html entities + // on the output that can be read by anyone. + + assert(document.root.tagName == "html"); // the root element is normal + + int foundCount; + // now let's loop through the whole tree + foreach(element; document.root.tree) { + // the asp thing will be in + if(auto asp = cast(AspCode) element) { + // you use the `asp.source` member to get the code for these + assert(asp.source == "% some asp code %"); + foundCount++; + } else if(element.tagName == "script") { + // and for raw source elements - script, style, or the ones you add, + // you use the innerHTML method to get the code inside + assert(element.innerHTML == "embedded && javascript"); + foundCount++; + } else if(element.tagName == "embedded-plaintext") { + // and innerHTML again + assert(element.innerHTML == "my plaintext & stuff"); + foundCount++; + } + + } + + assert(foundCount == 3); + + // writeln(document.toString()); +} + +// FIXME: