From fc037b4b825f0501a1489e10d7c822435d825cb7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 14 Jan 2023 16:57:10 +0100 Subject: [PATCH] Add support for incoming emails (#22056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #13585 fixes #9067 fixes #2386 ref #6226 ref #6219 fixes #745 This PR adds support to process incoming emails to perform actions. Currently I added handling of replies and unsubscribing from issues/pulls. In contrast to #13585 the IMAP IDLE command is used instead of polling which results (in my opinion πŸ˜‰) in cleaner code. Procedure: - When sending an issue/pull reply email, a token is generated which is present in the Reply-To and References header. - IMAP IDLE waits until a new email arrives - The token tells which action should be performed A possible signature and/or reply gets stripped from the content. I added a new service to the drone pipeline to test the receiving of incoming mails. If we keep this in, we may test our outgoing emails too in future. Co-authored-by: silverwind Co-authored-by: Lunny Xiao --- .drone.yml | 4 + assets/go-licenses.json | 25 ++ custom/conf/app.example.ini | 41 ++ .../doc/advanced/config-cheat-sheet.en-us.md | 14 + docs/content/doc/features/comparison.en-us.md | 2 +- docs/content/doc/features/comparison.zh-cn.md | 2 +- docs/content/doc/features/comparison.zh-tw.md | 2 +- .../content/doc/usage/incoming-email.en-us.md | 47 +++ go.mod | 7 +- go.sum | 24 +- models/unittest/testdb.go | 2 + modules/setting/incoming_email.go | 73 ++++ modules/setting/setting.go | 1 + modules/util/pack.go | 33 ++ modules/util/pack_test.go | 28 ++ routers/init.go | 2 + services/mailer/incoming/incoming.go | 375 ++++++++++++++++++ services/mailer/incoming/incoming_handler.go | 171 ++++++++ services/mailer/incoming/incoming_test.go | 138 +++++++ services/mailer/incoming/payload/payload.go | 70 ++++ services/mailer/mail.go | 55 ++- services/mailer/mail_test.go | 55 +-- services/mailer/mailer.go | 4 + services/mailer/token/token.go | 128 ++++++ tests/integration/incoming_email_test.go | 249 ++++++++++++ tests/mysql.ini.tmpl | 10 + 26 files changed, 1524 insertions(+), 38 deletions(-) create mode 100644 docs/content/doc/usage/incoming-email.en-us.md create mode 100644 modules/setting/incoming_email.go create mode 100644 modules/util/pack.go create mode 100644 modules/util/pack_test.go create mode 100644 services/mailer/incoming/incoming.go create mode 100644 services/mailer/incoming/incoming_handler.go create mode 100644 services/mailer/incoming/incoming_test.go create mode 100644 services/mailer/incoming/payload/payload.go create mode 100644 services/mailer/token/token.go create mode 100644 tests/integration/incoming_email_test.go diff --git a/.drone.yml b/.drone.yml index 2b5d21ddf967..d16386c7b49b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -230,6 +230,10 @@ services: MINIO_ACCESS_KEY: 123456 MINIO_SECRET_KEY: 12345678 + - name: smtpimap + image: tabascoterrier/docker-imap-devel:latest + pull: always + steps: - name: fetch-tags image: docker:git diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 6fd6a82e5088..30597388638f 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -209,6 +209,11 @@ "path": "github.com/caddyserver/certmagic/LICENSE.txt", "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, + { + "name": "github.com/cention-sany/utf7", + "path": "github.com/cention-sany/utf7/LICENSE", + "licenseText": "Copyright (c) 2013 The Go-IMAP Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n * Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the\n distribution.\n\n * Neither the name of the go-imap project nor the names of its\n contributors may be used to endorse or promote products derived\n from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/cespare/xxhash/v2", "path": "github.com/cespare/xxhash/v2/LICENSE.txt", @@ -254,6 +259,11 @@ "path": "github.com/dgryski/go-rendezvous/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2017-2020 Damian Gryski \u003cdamian@gryski.com\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" }, + { + "name": "github.com/dimiro1/reply", + "path": "github.com/dimiro1/reply/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) Discourse\nCopyright (c) Claudemiro\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/djherbis/buffer", "path": "github.com/djherbis/buffer/LICENSE.txt", @@ -284,6 +294,16 @@ "path": "github.com/editorconfig/editorconfig-core-go/v2/LICENSE", "licenseText": "MIT License\nCopyright (c) 2016 The Editorconfig Team\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/emersion/go-imap", + "path": "github.com/emersion/go-imap/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2013 The Go-IMAP Authors\nCopyright (c) 2016 emersion\nCopyright (c) 2016 Proton Technologies AG\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, + { + "name": "github.com/emersion/go-sasl", + "path": "github.com/emersion/go-sasl/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2016 emersion\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/ethantkoenig/rupture", "path": "github.com/ethantkoenig/rupture/LICENSE", @@ -524,6 +544,11 @@ "path": "github.com/jaytaylor/html2text/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2015 Jay Taylor\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n" }, + { + "name": "github.com/jhillyerd/enmime", + "path": "github.com/jhillyerd/enmime/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2012-2016 James Hillyerd, All Rights Reserved\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/josharian/intern", "path": "github.com/josharian/intern/license.md", diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index cec5e8cf0382..3233135e9da4 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1664,6 +1664,47 @@ ROUTER = console ;; convert \r\n to \n for Sendmail ;SENDMAIL_CONVERT_CRLF = true +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[email.incoming] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Enable handling of incoming emails. +;ENABLED = false +;; +;; The email address including the %{token} placeholder that will be replaced per user/action. +;; Example: incoming+%{token}@example.com +;; The placeholder must appear in the user part of the address (before the @). +;REPLY_TO_ADDRESS = +;; +;; IMAP server host +;HOST = +;; +;; IMAP server port +;PORT = +;; +;; Username of the receiving account +;USERNAME = +;; +;; Password of the receiving account +;PASSWORD = +;; +;; Whether the IMAP server uses TLS. +;USE_TLS = false +;; +;; If set to true, completely ignores server certificate validation errors. This option is unsafe. +;SKIP_TLS_VERIFY = true +;; +;; The mailbox name where incoming mail will end up. +;MAILBOX = INBOX +;; +;; Whether handled messages should be deleted from the mailbox. +;DELETE_HANDLED_MESSAGE = true +;; +;; Maximum size of a message to handle. Bigger messages are ignored. Set to 0 to allow every size. +;MAXIMUM_MESSAGE_SIZE = 10485760 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[cache] diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index def731472053..7dbcad6b40f6 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -750,6 +750,20 @@ and - `SEND_BUFFER_LEN`: **100**: Buffer length of mailing queue. **DEPRECATED** use `LENGTH` in `[queue.mailer]` - `SEND_AS_PLAIN_TEXT`: **false**: Send mails only in plain text, without HTML alternative. +## Incoming Email (`email.incoming`) + +- `ENABLED`: **false**: Enable handling of incoming emails. +- `REPLY_TO_ADDRESS`: **\**: The email address including the `%{token}` placeholder that will be replaced per user/action. Example: `incoming+%{token}@example.com`. The placeholder must appear in the user part of the address (before the `@`). +- `HOST`: **\**: IMAP server host. +- `PORT`: **\**: IMAP server port. +- `USERNAME`: **\**: Username of the receiving account. +- `PASSWORD`: **\**: Password of the receiving account. +- `USE_TLS`: **false**: Whether the IMAP server uses TLS. +- `SKIP_TLS_VERIFY`: **false**: If set to `true`, completely ignores server certificate validation errors. This option is unsafe. +- `MAILBOX`: **INBOX**: The mailbox name where incoming mail will end up. +- `DELETE_HANDLED_MESSAGE`: **true**: Whether handled messages should be deleted from the mailbox. +- `MAXIMUM_MESSAGE_SIZE`: **10485760**: Maximum size of a message to handle. Bigger messages are ignored. Set to 0 to allow every size. + ## Cache (`cache`) - `ENABLED`: **true**: Enable the cache. diff --git a/docs/content/doc/features/comparison.en-us.md b/docs/content/doc/features/comparison.en-us.md index 9baa6d512372..87b2f43de7d8 100644 --- a/docs/content/doc/features/comparison.en-us.md +++ b/docs/content/doc/features/comparison.en-us.md @@ -106,7 +106,7 @@ _Symbols used in table:_ | Issue search | βœ“ | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | Global issue search | [/](https://github.com/go-gitea/gitea/issues/2434) | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | Issue dependency | βœ“ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| Create issue via email | [✘](https://github.com/go-gitea/gitea/issues/6226) | ✘ | ✘ | ✘ | βœ“ | βœ“ | ✘ | +| Create issue via email | [✘](https://github.com/go-gitea/gitea/issues/6226) | ✘ | ✘ | βœ“ | βœ“ | βœ“ | ✘ | | Service Desk | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | βœ“ | βœ“ | ✘ | ✘ | ## Pull/Merge requests diff --git a/docs/content/doc/features/comparison.zh-cn.md b/docs/content/doc/features/comparison.zh-cn.md index aaf8eb4abc51..c5bec58cb85b 100644 --- a/docs/content/doc/features/comparison.zh-cn.md +++ b/docs/content/doc/features/comparison.zh-cn.md @@ -92,7 +92,7 @@ _θ‘¨ζ ΌδΈ­ηš„η¬¦ε·ε«δΉ‰:_ | ε·₯ε•ζœη΄’ | βœ“ | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | ε·₯ε•ε…¨ε±€ζœη΄’ | [✘](https://github.com/go-gitea/gitea/issues/2434) | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | ε·₯单依衖关系 | βœ“ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| ι€šθΏ‡ Email εˆ›ε»Ίε·₯单 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✘ | βœ“ | βœ“ | ✘ | +| ι€šθΏ‡ Email εˆ›ε»Ίε·₯单 | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | βœ“ | βœ“ | βœ“ | ✘ | | 服劑台 | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | [βœ“](https://gitlab.com/groups/gitlab-org/-/epics/3103) | βœ“ | ✘ | ✘ | #### Pull/Merge requests diff --git a/docs/content/doc/features/comparison.zh-tw.md b/docs/content/doc/features/comparison.zh-tw.md index 015955f0a852..4da9c74ec8cc 100644 --- a/docs/content/doc/features/comparison.zh-tw.md +++ b/docs/content/doc/features/comparison.zh-tw.md @@ -93,7 +93,7 @@ menu: | ε•ι‘Œζœε°‹ | βœ“ | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | ε…¨εŸŸε•ι‘Œζœε°‹ | [✘](https://github.com/go-gitea/gitea/issues/2434) | ✘ | βœ“ | βœ“ | βœ“ | βœ“ | ✘ | | ε•ι‘Œη›ΈδΎ | βœ“ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | -| εΎžι›»ε­ιƒ΅δ»Άε»Ίη«‹ε•ι‘Œ | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | ✘ | βœ“ | βœ“ | ✘ | +| εΎžι›»ε­ιƒ΅δ»Άε»Ίη«‹ε•ι‘Œ | [✘](https://github.com/go-gitea/gitea/issues/6226) | [✘](https://github.com/gogs/gogs/issues/2602) | ✘ | βœ“ | βœ“ | βœ“ | ✘ | | ζœε‹™ε° | [✘](https://github.com/go-gitea/gitea/issues/6219) | ✘ | ✘ | [βœ“](https://gitlab.com/groups/gitlab-org/-/epics/3103) | βœ“ | ✘ | ✘ | ## 拉取/εˆδ½΅θ«‹ζ±‚ diff --git a/docs/content/doc/usage/incoming-email.en-us.md b/docs/content/doc/usage/incoming-email.en-us.md new file mode 100644 index 000000000000..9f1f28e6681a --- /dev/null +++ b/docs/content/doc/usage/incoming-email.en-us.md @@ -0,0 +1,47 @@ +--- +date: "2022-12-01T00:00:00+00:00" +title: "Incoming Email" +slug: "incoming-email" +draft: false +toc: false +menu: + sidebar: + parent: "usage" + name: "Incoming Email" + weight: 13 + identifier: "incoming-email" +--- + +# Incoming Email + +Gitea supports the execution of several actions through incoming mails. This page describes how to set this up. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +Handling incoming email messages requires an IMAP-enabled email account. +The recommended strategy is to use [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing) but a catch-all mailbox does work too. +The receiving email address contains a user/action specific token which tells Gitea which action should be performed. +This token is expected in the `To` and `Delivered-To` header fields. + +Gitea tries to detect automatic responses to skip and the email server should be configured to reduce the incoming noise too (spam, newsletter). + +## Configuration + +To activate the handling of incoming email messages you have to configure the `email.incoming` section in the configuration file. + +The `REPLY_TO_ADDRESS` contains the address an email client will respond to. +This address needs to contain the `%{token}` placeholder which will be replaced with a token describing the user/action. +This placeholder must only appear once in the address and must be in the user part of the address (before the `@`). + +An example using email sub-addressing may look like this: `incoming+%{token}@example.com` + +If a catch-all mailbox is used, the placeholder may be used anywhere in the user part of the address: `incoming+%{token}@example.com`, `incoming_%{token}@example.com`, `%{token}@example.com` + +## Security + +Be careful when choosing the domain used for receiving incoming email. +It's recommended receiving incoming email on a subdomain, such as `incoming.example.com` to prevent potential security problems with other services running on `example.com`. diff --git a/go.mod b/go.mod index 889e54d321b0..b69f290ae1a5 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,13 @@ require ( github.com/buildkite/terminal-to-html/v3 v3.7.0 github.com/caddyserver/certmagic v0.17.2 github.com/chi-middleware/proxy v1.1.1 - github.com/denisenkom/go-mssqldb v0.12.3 + github.com/denisenkom/go-mssqldb v0.12.2 + github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 github.com/dustin/go-humanize v1.0.0 github.com/editorconfig/editorconfig-core-go/v2 v2.5.1 + github.com/emersion/go-imap v1.2.1 github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.3 @@ -58,6 +60,7 @@ require ( github.com/hashicorp/golang-lru v0.6.0 github.com/huandu/xstrings v1.4.0 github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba + github.com/jhillyerd/enmime v0.10.1 github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4 @@ -145,6 +148,7 @@ require ( github.com/blevesearch/zapx/v15 v15.3.8 // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect + github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cloudflare/circl v1.2.0 // indirect github.com/couchbase/go-couchbase v0.0.0-20210224140812-5740cd35f448 // indirect @@ -155,6 +159,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-ap/errors v0.0.0-20221205040414-01c1adfc98ea // indirect diff --git a/go.sum b/go.sum index 24e3ed735974..f8b9116e37f9 100644 --- a/go.sum +++ b/go.sum @@ -234,6 +234,8 @@ github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwM github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= +github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -294,17 +296,20 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= -github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= +github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw= +github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21 h1:PdsjTl0Cg+ZJgOx/CFV5NNgO1ThTreqdgKYiDCMHJwA= +github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21/go.mod h1:xJvkyD6Y2rZapGvPJLYo9dyx1s5dxBEDPa8T3YTuOk0= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= @@ -324,6 +329,12 @@ github.com/editorconfig/editorconfig-core-go/v2 v2.5.1 h1:EMpGLI+QHJMbvppCjIFTWu github.com/editorconfig/editorconfig-core-go/v2 v2.5.1/go.mod h1:9l0WF7U8RrFunzIpbUGLh1TIRUgDrfy0mpkyv8T7q9M= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA= +github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= +github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -456,6 +467,8 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/go-swagger/go-swagger v0.30.3 h1:HuzvdMRed/9Q8vmzVcfNBQByZVtT79DNZxZ18OprdoI= github.com/go-swagger/go-swagger v0.30.3/go.mod h1:neDPes8r8PCz2JPvHRDj8BTULLh4VJUt7n6MpQqxhHM= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013 h1:l9rI6sNaZgNC0LnF3MiE+qTmyBA/tZAg1rtyrGbUMK0= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU= github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA= github.com/go-webauthn/revoke v0.1.6 h1:3tv+itza9WpX5tryRQx4GwxCCBrCIiJ8GIkOhxiAmmU= @@ -497,6 +510,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= @@ -757,12 +771,15 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= +github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg= github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jhillyerd/enmime v0.10.1 h1:3VP8gFhK7R948YJBrna5bOgnTXEuPAoICo79kKkBKfA= +github.com/jhillyerd/enmime v0.10.1/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/z6mkcDHsA= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= @@ -876,6 +893,7 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -1064,6 +1082,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -1410,6 +1429,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 87d9304618b0..f3127006b8d3 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -106,6 +106,8 @@ func MainTest(m *testing.M, testOpts *TestOptions) { setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home") + setting.IncomingEmail.ReplyToAddress = "incoming+%{token}@localhost" + if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go new file mode 100644 index 000000000000..b6a637bccdc0 --- /dev/null +++ b/modules/setting/incoming_email.go @@ -0,0 +1,73 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "fmt" + "net/mail" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +var IncomingEmail = struct { + Enabled bool + ReplyToAddress string + TokenPlaceholder string `ini:"-"` + Host string + Port int + UseTLS bool `ini:"USE_TLS"` + SkipTLSVerify bool `ini:"SKIP_TLS_VERIFY"` + Username string + Password string + Mailbox string + DeleteHandledMessage bool + MaximumMessageSize uint32 +}{ + Mailbox: "INBOX", + DeleteHandledMessage: true, + TokenPlaceholder: "%{token}", + MaximumMessageSize: 10485760, +} + +func newIncomingEmail() { + if err := Cfg.Section("email.incoming").MapTo(&IncomingEmail); err != nil { + log.Fatal("Unable to map [email.incoming] section on to IncomingEmail. Error: %v", err) + } + + if !IncomingEmail.Enabled { + return + } + + if err := checkReplyToAddress(IncomingEmail.ReplyToAddress); err != nil { + log.Fatal("Invalid incoming_mail.REPLY_TO_ADDRESS (%s): %v", IncomingEmail.ReplyToAddress, err) + } +} + +func checkReplyToAddress(address string) error { + parsed, err := mail.ParseAddress(IncomingEmail.ReplyToAddress) + if err != nil { + return err + } + + if parsed.Name != "" { + return fmt.Errorf("name must not be set") + } + + c := strings.Count(IncomingEmail.ReplyToAddress, IncomingEmail.TokenPlaceholder) + switch c { + case 0: + return fmt.Errorf("%s must appear in the user part of the address (before the @)", IncomingEmail.TokenPlaceholder) + case 1: + default: + return fmt.Errorf("%s must appear only once", IncomingEmail.TokenPlaceholder) + } + + parts := strings.Split(IncomingEmail.ReplyToAddress, "@") + if !strings.Contains(parts[0], IncomingEmail.TokenPlaceholder) { + return fmt.Errorf("%s must appear in the user part of the address (before the @)", IncomingEmail.TokenPlaceholder) + } + + return nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index fa65b9489159..92861fb52867 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1341,6 +1341,7 @@ func NewServices() { newSessionService() newCORSService() parseMailerConfig(Cfg) + newIncomingEmail() newRegisterMailService() newNotifyMailService() newProxyService() diff --git a/modules/util/pack.go b/modules/util/pack.go new file mode 100644 index 000000000000..315d9f5066ff --- /dev/null +++ b/modules/util/pack.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "encoding/gob" +) + +// PackData uses gob to encode the given data in sequence +func PackData(data ...interface{}) ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + for _, datum := range data { + if err := enc.Encode(datum); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// UnpackData uses gob to decode the given data in sequence +func UnpackData(buf []byte, data ...interface{}) error { + r := bytes.NewReader(buf) + enc := gob.NewDecoder(r) + for _, datum := range data { + if err := enc.Decode(datum); err != nil { + return err + } + } + return nil +} diff --git a/modules/util/pack_test.go b/modules/util/pack_test.go new file mode 100644 index 000000000000..592c69cd0a32 --- /dev/null +++ b/modules/util/pack_test.go @@ -0,0 +1,28 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPackAndUnpackData(t *testing.T) { + s := "string" + i := int64(4) + f := float32(4.1) + + var s2 string + var i2 int64 + var f2 float32 + + data, err := PackData(s, i, f) + assert.NoError(t, err) + + assert.NoError(t, UnpackData(data, &s2, &i2, &f2)) + assert.NoError(t, UnpackData(data, &s2)) + assert.Error(t, UnpackData(data, &i2)) + assert.Error(t, UnpackData(data, &s2, &f2)) +} diff --git a/routers/init.go b/routers/init.go index 61af51222050..25f557d69569 100644 --- a/routers/init.go +++ b/routers/init.go @@ -40,6 +40,7 @@ import ( "code.gitea.io/gitea/services/automerge" "code.gitea.io/gitea/services/cron" "code.gitea.io/gitea/services/mailer" + mailer_incoming "code.gitea.io/gitea/services/mailer/incoming" markup_service "code.gitea.io/gitea/services/markup" repo_migrations "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" @@ -162,6 +163,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(task.Init) mustInit(repo_migrations.Init) eventsource.GetManager().Init() + mustInitCtx(ctx, mailer_incoming.Init) mustInitCtx(ctx, syncAppConfForGit) diff --git a/services/mailer/incoming/incoming.go b/services/mailer/incoming/incoming.go new file mode 100644 index 000000000000..2653e80586d7 --- /dev/null +++ b/services/mailer/incoming/incoming.go @@ -0,0 +1,375 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "context" + "crypto/tls" + "fmt" + net_mail "net/mail" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer/token" + + "github.com/dimiro1/reply" + "github.com/emersion/go-imap" + "github.com/emersion/go-imap/client" + "github.com/jhillyerd/enmime" +) + +var ( + addressTokenRegex *regexp.Regexp + referenceTokenRegex *regexp.Regexp +) + +func Init(ctx context.Context) error { + if !setting.IncomingEmail.Enabled { + return nil + } + + var err error + addressTokenRegex, err = regexp.Compile( + fmt.Sprintf( + `\A%s\z`, + strings.Replace(regexp.QuoteMeta(setting.IncomingEmail.ReplyToAddress), regexp.QuoteMeta(setting.IncomingEmail.TokenPlaceholder), "(.+)", 1), + ), + ) + if err != nil { + return err + } + referenceTokenRegex, err = regexp.Compile(fmt.Sprintf(`\Areply-(.+)@%s\z`, regexp.QuoteMeta(setting.Domain))) + if err != nil { + return err + } + + go func() { + ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Incoming Email", process.SystemProcessType, true) + defer finished() + + // This background job processes incoming emails. It uses the IMAP IDLE command to get notified about incoming emails. + // The following loop restarts the processing logic after errors until ctx indicates to stop. + + for { + select { + case <-ctx.Done(): + return + default: + if err := processIncomingEmails(ctx); err != nil { + log.Error("Error while processing incoming emails: %v", err) + } + select { + case <-ctx.Done(): + return + case <-time.NewTimer(10 * time.Second).C: + } + } + } + }() + + return nil +} + +// processIncomingEmails is the "main" method with the wait/process loop +func processIncomingEmails(ctx context.Context) error { + server := fmt.Sprintf("%s:%d", setting.IncomingEmail.Host, setting.IncomingEmail.Port) + + var c *client.Client + var err error + if setting.IncomingEmail.UseTLS { + c, err = client.DialTLS(server, &tls.Config{InsecureSkipVerify: setting.IncomingEmail.SkipTLSVerify}) + } else { + c, err = client.Dial(server) + } + if err != nil { + return fmt.Errorf("could not connect to server '%s': %w", server, err) + } + + if err := c.Login(setting.IncomingEmail.Username, setting.IncomingEmail.Password); err != nil { + return fmt.Errorf("could not login: %w", err) + } + defer func() { + if err := c.Logout(); err != nil { + log.Error("Logout from incoming email server failed: %v", err) + } + }() + + if _, err := c.Select(setting.IncomingEmail.Mailbox, false); err != nil { + return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err) + } + + // The following loop processes messages. If there are no messages available, IMAP IDLE is used to wait for new messages. + // This process is repeated until an IMAP error occurs or ctx indicates to stop. + + for { + select { + case <-ctx.Done(): + return nil + default: + if err := processMessages(ctx, c); err != nil { + return fmt.Errorf("could not process messages: %w", err) + } + if err := waitForUpdates(ctx, c); err != nil { + return fmt.Errorf("wait for updates failed: %w", err) + } + select { + case <-ctx.Done(): + return nil + case <-time.NewTimer(time.Second).C: + } + } + } +} + +// waitForUpdates uses IMAP IDLE to wait for new emails +func waitForUpdates(ctx context.Context, c *client.Client) error { + updates := make(chan client.Update, 1) + + c.Updates = updates + defer func() { + c.Updates = nil + }() + + errs := make(chan error, 1) + stop := make(chan struct{}) + go func() { + errs <- c.Idle(stop, nil) + }() + + stopped := false + for { + select { + case update := <-updates: + switch update.(type) { + case *client.MailboxUpdate: + if !stopped { + close(stop) + stopped = true + } + default: + } + case err := <-errs: + if err != nil { + return fmt.Errorf("imap idle failed: %w", err) + } + return nil + case <-ctx.Done(): + return nil + } + } +} + +// processMessages searches unread mails and processes them. +func processMessages(ctx context.Context, c *client.Client) error { + criteria := imap.NewSearchCriteria() + criteria.WithoutFlags = []string{imap.SeenFlag} + criteria.Smaller = setting.IncomingEmail.MaximumMessageSize + ids, err := c.Search(criteria) + if err != nil { + return fmt.Errorf("imap search failed: %w", err) + } + + if len(ids) == 0 { + return nil + } + + seqset := new(imap.SeqSet) + seqset.AddNum(ids...) + messages := make(chan *imap.Message, 10) + + section := &imap.BodySectionName{} + + errs := make(chan error, 1) + go func() { + errs <- c.Fetch( + seqset, + []imap.FetchItem{section.FetchItem()}, + messages, + ) + }() + + handledSet := new(imap.SeqSet) +loop: + for { + select { + case <-ctx.Done(): + break loop + case msg, ok := <-messages: + if !ok { + if setting.IncomingEmail.DeleteHandledMessage && !handledSet.Empty() { + if err := c.Store( + handledSet, + imap.FormatFlagsOp(imap.AddFlags, true), + []interface{}{imap.DeletedFlag}, + nil, + ); err != nil { + return fmt.Errorf("imap store failed: %w", err) + } + + if err := c.Expunge(nil); err != nil { + return fmt.Errorf("imap expunge failed: %w", err) + } + } + return nil + } + + err := func() error { + r := msg.GetBody(section) + if r == nil { + return fmt.Errorf("could not get body from message: %w", err) + } + + env, err := enmime.ReadEnvelope(r) + if err != nil { + return fmt.Errorf("could not read envelope: %w", err) + } + + if isAutomaticReply(env) { + log.Debug("Skipping automatic email reply") + return nil + } + + t := searchTokenInHeaders(env) + if t == "" { + log.Debug("Incoming email token not found in headers") + return nil + } + + handlerType, user, payload, err := token.ExtractToken(ctx, t) + if err != nil { + if _, ok := err.(*token.ErrToken); ok { + log.Info("Invalid incoming email token: %v", err) + return nil + } + return err + } + + handler, ok := handlers[handlerType] + if !ok { + return fmt.Errorf("unexpected handler type: %v", handlerType) + } + + content := getContentFromMailReader(env) + + if err := handler.Handle(ctx, content, user, payload); err != nil { + return fmt.Errorf("could not handle message: %w", err) + } + + handledSet.AddNum(msg.SeqNum) + + return nil + }() + if err != nil { + log.Error("Error while processing incoming email[%v]: %v", msg.Uid, err) + } + } + } + + if err := <-errs; err != nil { + return fmt.Errorf("imap fetch failed: %w", err) + } + + return nil +} + +// isAutomaticReply tests if the headers indicate an automatic reply +func isAutomaticReply(env *enmime.Envelope) bool { + autoSubmitted := env.GetHeader("Auto-Submitted") + if autoSubmitted != "" && autoSubmitted != "no" { + return true + } + autoReply := env.GetHeader("X-Autoreply") + if autoReply == "yes" { + return true + } + autoRespond := env.GetHeader("X-Autorespond") + return autoRespond != "" +} + +// searchTokenInHeaders looks for the token in To, Delivered-To and References +func searchTokenInHeaders(env *enmime.Envelope) string { + if addressTokenRegex != nil { + to, _ := env.AddressList("To") + + token := searchTokenInAddresses(to) + if token != "" { + return token + } + + deliveredTo, _ := env.AddressList("Delivered-To") + + token = searchTokenInAddresses(deliveredTo) + if token != "" { + return token + } + } + + references := env.GetHeader("References") + for { + begin := strings.IndexByte(references, '<') + if begin == -1 { + break + } + begin++ + + end := strings.IndexByte(references, '>') + if end == -1 || begin > end { + break + } + + match := referenceTokenRegex.FindStringSubmatch(references[begin:end]) + if len(match) == 2 { + return match[1] + } + + references = references[end+1:] + } + + return "" +} + +// searchTokenInAddresses looks for the token in an address +func searchTokenInAddresses(addresses []*net_mail.Address) string { + for _, address := range addresses { + match := addressTokenRegex.FindStringSubmatch(address.Address) + if len(match) != 2 { + continue + } + + return match[1] + } + + return "" +} + +type MailContent struct { + Content string + Attachments []*Attachment +} + +type Attachment struct { + Name string + Content []byte +} + +// getContentFromMailReader grabs the plain content and the attachments from the mail. +// A potential reply/signature gets stripped from the content. +func getContentFromMailReader(env *enmime.Envelope) *MailContent { + attachments := make([]*Attachment, 0, len(env.Attachments)) + for _, attachment := range env.Attachments { + attachments = append(attachments, &Attachment{ + Name: attachment.FileName, + Content: attachment.Content, + }) + } + + return &MailContent{ + Content: reply.FromText(env.Text), + Attachments: attachments, + } +} diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go new file mode 100644 index 000000000000..173b362a5505 --- /dev/null +++ b/services/mailer/incoming/incoming_handler.go @@ -0,0 +1,171 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "bytes" + "context" + "fmt" + + issues_model "code.gitea.io/gitea/models/issues" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/upload" + "code.gitea.io/gitea/modules/util" + attachment_service "code.gitea.io/gitea/services/attachment" + issue_service "code.gitea.io/gitea/services/issue" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + "code.gitea.io/gitea/services/mailer/token" + pull_service "code.gitea.io/gitea/services/pull" +) + +type MailHandler interface { + Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error +} + +var handlers = map[token.HandlerType]MailHandler{ + token.ReplyHandlerType: &ReplyHandler{}, + token.UnsubscribeHandlerType: &UnsubscribeHandler{}, +} + +// ReplyHandler handles incoming emails to create a reply from them +type ReplyHandler struct{} + +func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error { + if doer == nil { + return util.NewInvalidArgumentErrorf("doer can't be nil") + } + + ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) + if err != nil { + return err + } + + var issue *issues_model.Issue + + switch r := ref.(type) { + case *issues_model.Issue: + issue = r + case *issues_model.Comment: + comment := r + + if err := comment.LoadIssue(ctx); err != nil { + return err + } + + issue = comment.Issue + default: + return util.NewInvalidArgumentErrorf("unsupported reply reference: %v", ref) + } + + if err := issue.LoadRepo(ctx); err != nil { + return err + } + + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer) + if err != nil { + return err + } + + if !perm.CanWriteIssuesOrPulls(issue.IsPull) || issue.IsLocked && !doer.IsAdmin { + log.Debug("can't write issue or pull") + return nil + } + + switch r := ref.(type) { + case *issues_model.Issue: + attachmentIDs := make([]string, 0, len(content.Attachments)) + if setting.Attachment.Enabled { + for _, attachment := range content.Attachments { + a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{ + Name: attachment.Name, + UploaderID: doer.ID, + RepoID: issue.Repo.ID, + }) + if err != nil { + if upload.IsErrFileTypeForbidden(err) { + log.Info("Skipping disallowed attachment type: %s", attachment.Name) + continue + } + return err + } + attachmentIDs = append(attachmentIDs, a.UUID) + } + } + + if content.Content == "" && len(attachmentIDs) == 0 { + return nil + } + + _, err = issue_service.CreateIssueComment(ctx, doer, issue.Repo, issue, content.Content, attachmentIDs) + if err != nil { + return fmt.Errorf("CreateIssueComment failed: %w", err) + } + case *issues_model.Comment: + comment := r + + if content.Content == "" { + return nil + } + + if comment.Type == issues_model.CommentTypeCode { + _, err := pull_service.CreateCodeComment( + ctx, + doer, + nil, + issue, + comment.Line, + content.Content, + comment.TreePath, + false, + comment.ReviewID, + "", + ) + if err != nil { + return fmt.Errorf("CreateCodeComment failed: %w", err) + } + } + } + return nil +} + +// UnsubscribeHandler handles unwatching issues/pulls +type UnsubscribeHandler struct{} + +func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, doer *user_model.User, payload []byte) error { + if doer == nil { + return util.NewInvalidArgumentErrorf("doer can't be nil") + } + + ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload) + if err != nil { + return err + } + + switch r := ref.(type) { + case *issues_model.Issue: + issue := r + + if err := issue.LoadRepo(ctx); err != nil { + return err + } + + perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer) + if err != nil { + return err + } + + if !perm.CanReadIssuesOrPulls(issue.IsPull) { + log.Debug("can't read issue or pull") + return nil + } + + return issues_model.CreateOrUpdateIssueWatch(doer.ID, issue.ID, false) + } + + return fmt.Errorf("unsupported unsubscribe reference: %v", ref) +} diff --git a/services/mailer/incoming/incoming_test.go b/services/mailer/incoming/incoming_test.go new file mode 100644 index 000000000000..5d84848e3f45 --- /dev/null +++ b/services/mailer/incoming/incoming_test.go @@ -0,0 +1,138 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package incoming + +import ( + "strings" + "testing" + + "github.com/jhillyerd/enmime" + "github.com/stretchr/testify/assert" +) + +func TestIsAutomaticReply(t *testing.T) { + cases := []struct { + Headers map[string]string + Expected bool + }{ + { + Headers: map[string]string{}, + Expected: false, + }, + { + Headers: map[string]string{ + "Auto-Submitted": "no", + }, + Expected: false, + }, + { + Headers: map[string]string{ + "Auto-Submitted": "yes", + }, + Expected: true, + }, + { + Headers: map[string]string{ + "X-Autoreply": "no", + }, + Expected: false, + }, + { + Headers: map[string]string{ + "X-Autoreply": "yes", + }, + Expected: true, + }, + { + Headers: map[string]string{ + "X-Autorespond": "yes", + }, + Expected: true, + }, + } + + for _, c := range cases { + b := enmime.Builder(). + From("Dummy", "dummy@gitea.io"). + To("Dummy", "dummy@gitea.io") + for k, v := range c.Headers { + b = b.Header(k, v) + } + root, err := b.Build() + assert.NoError(t, err) + env, err := enmime.EnvelopeFromPart(root) + assert.NoError(t, err) + + assert.Equal(t, c.Expected, isAutomaticReply(env)) + } +} + +func TestGetContentFromMailReader(t *testing.T) { + mailString := "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "mail content\r\n" + + "--text-boundary--\r\n" + + "--message-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: attachment; filename=attachment.txt\r\n" + + "\r\n" + + "attachment content\r\n" + + "--message-boundary--\r\n" + + env, err := enmime.ReadEnvelope(strings.NewReader(mailString)) + assert.NoError(t, err) + content := getContentFromMailReader(env) + assert.Equal(t, "mail content", content.Content) + assert.Len(t, content.Attachments, 1) + assert.Equal(t, "attachment.txt", content.Attachments[0].Name) + assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content) + + mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/html\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "

mail content

\r\n" + + "--text-boundary--\r\n" + + "--message-boundary--\r\n" + + env, err = enmime.ReadEnvelope(strings.NewReader(mailString)) + assert.NoError(t, err) + content = getContentFromMailReader(env) + assert.Equal(t, "mail content", content.Content) + assert.Empty(t, content.Attachments) + + mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" + + "\r\n" + + "--message-boundary\r\n" + + "Content-Type: multipart/alternative; boundary=text-boundary\r\n" + + "\r\n" + + "--text-boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Disposition: inline\r\n" + + "\r\n" + + "mail content without signature\r\n" + + "--\r\n" + + "signature\r\n" + + "--text-boundary--\r\n" + + "--message-boundary--\r\n" + + env, err = enmime.ReadEnvelope(strings.NewReader(mailString)) + assert.NoError(t, err) + content = getContentFromMailReader(env) + assert.NoError(t, err) + assert.Equal(t, "mail content without signature", content.Content) + assert.Empty(t, content.Attachments) +} diff --git a/services/mailer/incoming/payload/payload.go b/services/mailer/incoming/payload/payload.go new file mode 100644 index 000000000000..eb82f5c3ed32 --- /dev/null +++ b/services/mailer/incoming/payload/payload.go @@ -0,0 +1,70 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package payload + +import ( + "context" + + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/util" +) + +const replyPayloadVersion1 byte = 1 + +type payloadReferenceType byte + +const ( + payloadReferenceIssue payloadReferenceType = iota + payloadReferenceComment +) + +// CreateReferencePayload creates data which GetReferenceFromPayload resolves to the reference again. +func CreateReferencePayload(reference interface{}) ([]byte, error) { + var refType payloadReferenceType + var refID int64 + + switch r := reference.(type) { + case *issues_model.Issue: + refType = payloadReferenceIssue + refID = r.ID + case *issues_model.Comment: + refType = payloadReferenceComment + refID = r.ID + default: + return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", r) + } + + payload, err := util.PackData(refType, refID) + if err != nil { + return nil, err + } + + return append([]byte{replyPayloadVersion1}, payload...), nil +} + +// GetReferenceFromPayload resolves the reference from the payload +func GetReferenceFromPayload(ctx context.Context, payload []byte) (interface{}, error) { + if len(payload) < 1 { + return nil, util.NewInvalidArgumentErrorf("payload to small") + } + + if payload[0] != replyPayloadVersion1 { + return nil, util.NewInvalidArgumentErrorf("unsupported payload version") + } + + var ref payloadReferenceType + var id int64 + if err := util.UnpackData(payload[1:], &ref, &id); err != nil { + return nil, err + } + + switch ref { + case payloadReferenceIssue: + return issues_model.GetIssueByID(ctx, id) + case payloadReferenceComment: + return issues_model.GetCommentByID(ctx, id) + default: + return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", ref) + } +} diff --git a/services/mailer/mail.go b/services/mailer/mail.go index 674011bede2a..6af4ed249c5b 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -29,6 +29,8 @@ import ( "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/translation" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + "code.gitea.io/gitea/services/mailer/token" "gopkg.in/gomail.v2" ) @@ -302,14 +304,57 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient msgID := createReference(ctx.Issue, ctx.Comment, ctx.ActionType) reference := createReference(ctx.Issue, nil, activities_model.ActionType(0)) + var replyPayload []byte + if ctx.Comment != nil && ctx.Comment.Type == issues_model.CommentTypeCode { + replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Comment) + } else { + replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Issue) + } + if err != nil { + return nil, err + } + + unsubscribePayload, err := incoming_payload.CreateReferencePayload(ctx.Issue) + if err != nil { + return nil, err + } + msgs := make([]*Message, 0, len(recipients)) for _, recipient := range recipients { msg := NewMessageFrom([]string{recipient.Email}, ctx.Doer.DisplayName(), setting.MailService.FromEmail, subject, mailBody.String()) msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info) - msg.SetHeader("Message-ID", "<"+msgID+">") - msg.SetHeader("In-Reply-To", "<"+reference+">") - msg.SetHeader("References", "<"+reference+">") + msg.SetHeader("Message-ID", msgID) + msg.SetHeader("In-Reply-To", reference) + + references := []string{reference} + listUnsubscribe := []string{"<" + ctx.Issue.HTMLURL() + ">"} + + if setting.IncomingEmail.Enabled { + if ctx.Comment != nil { + token, err := token.CreateToken(token.ReplyHandlerType, recipient, replyPayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + replyAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + msg.ReplyTo = replyAddress + msg.SetHeader("List-Post", fmt.Sprintf("", replyAddress)) + + references = append(references, fmt.Sprintf("", token, setting.Domain)) + } + } + + token, err := token.CreateToken(token.UnsubscribeHandlerType, recipient, unsubscribePayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + unsubAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + listUnsubscribe = append(listUnsubscribe, "") + } + } + + msg.SetHeader("References", references...) + msg.SetHeader("List-Unsubscribe", listUnsubscribe...) for key, value := range generateAdditionalHeaders(ctx, actType, recipient) { msg.SetHeader(key, value) @@ -345,7 +390,7 @@ func createReference(issue *issues_model.Issue, comment *issues_model.Comment, a } } - return fmt.Sprintf("%s/%s/%d%s@%s", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) + return fmt.Sprintf("<%s/%s/%d%s@%s>", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) } func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string { @@ -357,8 +402,6 @@ func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient // https://datatracker.ietf.org/doc/html/rfc2369 "List-Archive": fmt.Sprintf("<%s>", repo.HTMLURL()), - //"List-Post": https://github.com/go-gitea/gitea/pull/13585 - "List-Unsubscribe": ctx.Issue.HTMLURL(), "X-Mailer": "Gitea", "X-Gitea-Reason": reason, diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 6ed4fed9bdad..64f2f740ca2d 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "html/template" + "regexp" "strings" "testing" texttmpl "text/template" @@ -66,6 +67,9 @@ func prepareMailerTest(t *testing.T) (doer *user_model.User, repo *repo_model.Re func TestComposeIssueCommentMessage(t *testing.T) { doer, _, issue, comment := prepareMailerTest(t) + setting.IncomingEmail.Enabled = true + defer func() { setting.IncomingEmail.Enabled = false }() + subjectTemplates = texttmpl.Must(texttmpl.New("issue/comment").Parse(subjectTpl)) bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl)) @@ -78,18 +82,20 @@ func TestComposeIssueCommentMessage(t *testing.T) { assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() - mailto := gomailMsg.GetHeader("To") - subject := gomailMsg.GetHeader("Subject") - messageID := gomailMsg.GetHeader("Message-ID") - inReplyTo := gomailMsg.GetHeader("In-Reply-To") - references := gomailMsg.GetHeader("References") + replyTo := gomailMsg.GetHeader("Reply-To")[0] + subject := gomailMsg.GetHeader("Subject")[0] - assert.Len(t, mailto, 1, "exactly one recipient is expected in the To field") - assert.Equal(t, "Re: ", subject[0][:4], "Comment reply subject should contain Re:") - assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject[0]) - assert.Equal(t, "", inReplyTo[0], "In-Reply-To header doesn't match") - assert.Equal(t, "", references[0], "References header doesn't match") - assert.Equal(t, "", messageID[0], "Message-ID header doesn't match") + assert.Len(t, gomailMsg.GetHeader("To"), 1, "exactly one recipient is expected in the To field") + tokenRegex := regexp.MustCompile(`\Aincoming\+(.+)@localhost\z`) + assert.Regexp(t, tokenRegex, replyTo) + token := tokenRegex.FindAllStringSubmatch(replyTo, 1)[0][1] + assert.Equal(t, "Re: ", subject[:4], "Comment reply subject should contain Re:") + assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject) + assert.Equal(t, "", gomailMsg.GetHeader("In-Reply-To")[0], "In-Reply-To header doesn't match") + assert.ElementsMatch(t, []string{"", ""}, gomailMsg.GetHeader("References"), "References header doesn't match") + assert.Equal(t, "", gomailMsg.GetHeader("Message-ID")[0], "Message-ID header doesn't match") + assert.Equal(t, "", gomailMsg.GetHeader("List-Post")[0]) + assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 2) // url + mailto } func TestComposeIssueMessage(t *testing.T) { @@ -119,6 +125,8 @@ func TestComposeIssueMessage(t *testing.T) { assert.Equal(t, "", inReplyTo[0], "In-Reply-To header doesn't match") assert.Equal(t, "", references[0], "References header doesn't match") assert.Equal(t, "", messageID[0], "Message-ID header doesn't match") + assert.Empty(t, gomailMsg.GetHeader("List-Post")) // incoming mail feature disabled + assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 1) // url without mailto } func TestTemplateSelection(t *testing.T) { @@ -238,7 +246,6 @@ func TestGenerateAdditionalHeaders(t *testing.T) { expected := map[string]string{ "List-ID": "user2/repo1 ", "List-Archive": "", - "List-Unsubscribe": "https://try.gitea.io/user2/repo1/issues/1", "X-Gitea-Reason": "dummy-reason", "X-Gitea-Sender": "< Ur Tw ><", "X-Gitea-Recipient": "Test", @@ -271,7 +278,6 @@ func Test_createReference(t *testing.T) { name string args args prefix string - suffix string }{ { name: "Open Issue", @@ -279,7 +285,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionCreateIssue, }, - prefix: fmt.Sprintf("%s/issues/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain), + prefix: fmt.Sprintf("<%s/issues/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain), }, { name: "Open Pull", @@ -287,7 +293,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionCreatePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain), + prefix: fmt.Sprintf("<%s/pulls/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain), }, { name: "Comment Issue", @@ -296,7 +302,7 @@ func Test_createReference(t *testing.T) { comment: comment, actionType: activities_model.ActionCommentIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), + prefix: fmt.Sprintf("<%s/issues/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), }, { name: "Comment Pull", @@ -305,7 +311,7 @@ func Test_createReference(t *testing.T) { comment: comment, actionType: activities_model.ActionCommentPull, }, - prefix: fmt.Sprintf("%s/pulls/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), + prefix: fmt.Sprintf("<%s/pulls/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain), }, { name: "Close Issue", @@ -313,7 +319,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionCloseIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/close/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/issues/%d/close/", issue.Repo.FullName(), issue.Index), }, { name: "Close Pull", @@ -321,7 +327,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionClosePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index), }, { name: "Reopen Issue", @@ -329,7 +335,7 @@ func Test_createReference(t *testing.T) { issue: issue, actionType: activities_model.ActionReopenIssue, }, - prefix: fmt.Sprintf("%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index), }, { name: "Reopen Pull", @@ -337,7 +343,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionReopenPullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index), }, { name: "Merge Pull", @@ -345,7 +351,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionMergePullRequest, }, - prefix: fmt.Sprintf("%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index), }, { name: "Ready Pull", @@ -353,7 +359,7 @@ func Test_createReference(t *testing.T) { issue: pullIssue, actionType: activities_model.ActionPullRequestReadyForReview, }, - prefix: fmt.Sprintf("%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index), + prefix: fmt.Sprintf("<%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index), }, } for _, tt := range tests { @@ -362,9 +368,6 @@ func Test_createReference(t *testing.T) { if !strings.HasPrefix(got, tt.prefix) { t.Errorf("createReference() = %v, want %v", got, tt.prefix) } - if !strings.HasSuffix(got, tt.suffix) { - t.Errorf("createReference() = %v, want %v", got, tt.prefix) - } }) } } diff --git a/services/mailer/mailer.go b/services/mailer/mailer.go index 6149644600ea..4e03afb96142 100644 --- a/services/mailer/mailer.go +++ b/services/mailer/mailer.go @@ -36,6 +36,7 @@ type Message struct { FromAddress string FromDisplayName string To []string + ReplyTo string Subject string Date time.Time Body string @@ -47,6 +48,9 @@ func (m *Message) ToMessage() *gomail.Message { msg := gomail.NewMessage() msg.SetAddressHeader("From", m.FromAddress, m.FromDisplayName) msg.SetHeader("To", m.To...) + if m.ReplyTo != "" { + msg.SetHeader("Reply-To", m.ReplyTo) + } for header := range m.Headers { msg.SetHeader(header, m.Headers[header]...) } diff --git a/services/mailer/token/token.go b/services/mailer/token/token.go new file mode 100644 index 000000000000..8a5a762d6b5f --- /dev/null +++ b/services/mailer/token/token.go @@ -0,0 +1,128 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package token + +import ( + "context" + crypto_hmac "crypto/hmac" + "crypto/sha256" + "encoding/base32" + "fmt" + "time" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" +) + +// A token is a verifiable container describing an action. +// +// A token has a dynamic length depending on the contained data and has the following structure: +// | Token Version | User ID | HMAC | Payload | +// +// The payload is verifiable by the generated HMAC using the user secret. It contains: +// | Timestamp | Action/Handler Type | Action/Handler Data | + +const ( + tokenVersion1 byte = 1 + tokenLifetimeInYears int = 1 +) + +type HandlerType byte + +const ( + UnknownHandlerType HandlerType = iota + ReplyHandlerType + UnsubscribeHandlerType +) + +var encodingWithoutPadding = base32.StdEncoding.WithPadding(base32.NoPadding) + +type ErrToken struct { + context string +} + +func (err *ErrToken) Error() string { + return "invalid email token: " + err.context +} + +func (err *ErrToken) Unwrap() error { + return util.ErrInvalidArgument +} + +// CreateToken creates a token for the action/user tuple +func CreateToken(ht HandlerType, user *user_model.User, data []byte) (string, error) { + payload, err := util.PackData( + time.Now().AddDate(tokenLifetimeInYears, 0, 0).Unix(), + ht, + data, + ) + if err != nil { + return "", err + } + + packagedData, err := util.PackData( + user.ID, + generateHmac([]byte(user.Rands), payload), + payload, + ) + if err != nil { + return "", err + } + + return encodingWithoutPadding.EncodeToString(append([]byte{tokenVersion1}, packagedData...)), nil +} + +// ExtractToken extracts the action/user tuple from the token and verifies the content +func ExtractToken(ctx context.Context, token string) (HandlerType, *user_model.User, []byte, error) { + data, err := encodingWithoutPadding.DecodeString(token) + if err != nil { + return UnknownHandlerType, nil, nil, err + } + + if len(data) < 1 { + return UnknownHandlerType, nil, nil, &ErrToken{"no data"} + } + + if data[0] != tokenVersion1 { + return UnknownHandlerType, nil, nil, &ErrToken{fmt.Sprintf("unsupported token version: %v", data[0])} + } + + var userID int64 + var hmac []byte + var payload []byte + if err := util.UnpackData(data[1:], &userID, &hmac, &payload); err != nil { + return UnknownHandlerType, nil, nil, err + } + + user, err := user_model.GetUserByID(ctx, userID) + if err != nil { + return UnknownHandlerType, nil, nil, err + } + + if !crypto_hmac.Equal(hmac, generateHmac([]byte(user.Rands), payload)) { + return UnknownHandlerType, nil, nil, &ErrToken{"verification failed"} + } + + var expiresUnix int64 + var handlerType HandlerType + var innerPayload []byte + if err := util.UnpackData(payload, &expiresUnix, &handlerType, &innerPayload); err != nil { + return UnknownHandlerType, nil, nil, err + } + + if time.Unix(expiresUnix, 0).Before(time.Now()) { + return UnknownHandlerType, nil, nil, &ErrToken{"token expired"} + } + + return handlerType, user, innerPayload, nil +} + +// generateHmac creates a trunkated HMAC for the given payload +func generateHmac(secret, payload []byte) []byte { + mac := crypto_hmac.New(sha256.New, secret) + mac.Write(payload) + hmac := mac.Sum(nil) + + return hmac[:10] // RFC2104 recommends not using less then 80 bits +} diff --git a/tests/integration/incoming_email_test.go b/tests/integration/incoming_email_test.go new file mode 100644 index 000000000000..b4478f57809f --- /dev/null +++ b/tests/integration/incoming_email_test.go @@ -0,0 +1,249 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "io" + "net" + "net/smtp" + "strings" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer/incoming" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + token_service "code.gitea.io/gitea/services/mailer/token" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" + "gopkg.in/gomail.v2" +) + +func TestIncomingEmail(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1}) + + t.Run("Payload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 1}) + + _, err := incoming_payload.CreateReferencePayload(user) + assert.Error(t, err) + + issuePayload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + commentPayload, err := incoming_payload.CreateReferencePayload(comment) + assert.NoError(t, err) + + _, err = incoming_payload.GetReferenceFromPayload(db.DefaultContext, []byte{1, 2, 3}) + assert.Error(t, err) + + ref, err := incoming_payload.GetReferenceFromPayload(db.DefaultContext, issuePayload) + assert.NoError(t, err) + assert.IsType(t, ref, new(issues_model.Issue)) + assert.EqualValues(t, issue.ID, ref.(*issues_model.Issue).ID) + + ref, err = incoming_payload.GetReferenceFromPayload(db.DefaultContext, commentPayload) + assert.NoError(t, err) + assert.IsType(t, ref, new(issues_model.Comment)) + assert.EqualValues(t, comment.ID, ref.(*issues_model.Comment).ID) + }) + + t.Run("Token", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + payload := []byte{1, 2, 3, 4, 5} + + token, err := token_service.CreateToken(token_service.ReplyHandlerType, user, payload) + assert.NoError(t, err) + assert.NotEmpty(t, token) + + ht, u, p, err := token_service.ExtractToken(db.DefaultContext, token) + assert.NoError(t, err) + assert.Equal(t, token_service.ReplyHandlerType, ht) + assert.Equal(t, user.ID, u.ID) + assert.Equal(t, payload, p) + }) + + t.Run("Handler", func(t *testing.T) { + t.Run("Reply", func(t *testing.T) { + t.Run("Comment", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + handler := &incoming.ReplyHandler{} + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + + assert.Error(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, nil, payload)) + assert.NoError(t, handler.Handle(db.DefaultContext, &incoming.MailContent{}, user, payload)) + + content := &incoming.MailContent{ + Content: "reply by mail", + Attachments: []*incoming.Attachment{ + { + Name: "attachment.txt", + Content: []byte("test"), + }, + }, + } + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeComment, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + comment := comments[len(comments)-1] + assert.Equal(t, user.ID, comment.PosterID) + assert.Equal(t, content.Content, comment.Content) + assert.NoError(t, comment.LoadAttachments(db.DefaultContext)) + assert.Len(t, comment.Attachments, 1) + attachment := comment.Attachments[0] + assert.Equal(t, content.Attachments[0].Name, attachment.Name) + assert.EqualValues(t, 4, attachment.Size) + }) + + t.Run("CodeComment", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 6}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}) + + handler := &incoming.ReplyHandler{} + content := &incoming.MailContent{ + Content: "code reply by mail", + Attachments: []*incoming.Attachment{ + { + Name: "attachment.txt", + Content: []byte("test"), + }, + }, + } + + payload, err := incoming_payload.CreateReferencePayload(comment) + assert.NoError(t, err) + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeCode, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + comment = comments[len(comments)-1] + assert.Equal(t, user.ID, comment.PosterID) + assert.Equal(t, content.Content, comment.Content) + assert.NoError(t, comment.LoadAttachments(db.DefaultContext)) + assert.Empty(t, comment.Attachments) + }) + }) + + t.Run("Unsubscribe", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + watching, err := issues_model.CheckIssueWatch(user, issue) + assert.NoError(t, err) + assert.True(t, watching) + + handler := &incoming.UnsubscribeHandler{} + + content := &incoming.MailContent{ + Content: "unsub me", + } + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + + assert.NoError(t, handler.Handle(db.DefaultContext, content, user, payload)) + + watching, err = issues_model.CheckIssueWatch(user, issue) + assert.NoError(t, err) + assert.False(t, watching) + }) + }) + + if setting.IncomingEmail.Enabled { + // This test connects to the configured email server and is currently only enabled for MySql integration tests. + // It sends a reply to create a comment. If the comment is not detected after 10 seconds the test fails. + t.Run("IMAP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + payload, err := incoming_payload.CreateReferencePayload(issue) + assert.NoError(t, err) + token, err := token_service.CreateToken(token_service.ReplyHandlerType, user, payload) + assert.NoError(t, err) + + msg := gomail.NewMessage() + msg.SetHeader("To", strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1)) + msg.SetHeader("From", user.Email) + msg.SetBody("text/plain", token) + err = gomail.Send(&smtpTestSender{}, msg) + assert.NoError(t, err) + + assert.Eventually(t, func() bool { + comments, err := issues_model.FindComments(db.DefaultContext, &issues_model.FindCommentsOptions{ + IssueID: issue.ID, + Type: issues_model.CommentTypeComment, + }) + assert.NoError(t, err) + assert.NotEmpty(t, comments) + + comment := comments[len(comments)-1] + + return comment.PosterID == user.ID && comment.Content == token + }, 10*time.Second, 1*time.Second) + }) + } +} + +// A simple SMTP mail sender used for integration tests. +type smtpTestSender struct{} + +func (s *smtpTestSender) Send(from string, to []string, msg io.WriterTo) error { + conn, err := net.Dial("tcp", net.JoinHostPort(setting.IncomingEmail.Host, "25")) + if err != nil { + return err + } + defer conn.Close() + + client, err := smtp.NewClient(conn, setting.IncomingEmail.Host) + if err != nil { + return err + } + + if err = client.Mail(from); err != nil { + return err + } + + for _, rec := range to { + if err = client.Rcpt(rec); err != nil { + return err + } + } + + w, err := client.Data() + if err != nil { + return err + } + if _, err := msg.WriteTo(w); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + + return client.Quit() +} diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 24a9a02dc46c..44914d087962 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -124,3 +124,13 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [packages] ENABLED = true + +[email.incoming] +ENABLED = true +HOST = smtpimap +PORT = 993 +USERNAME = debug@localdomain.test +PASSWORD = debug +USE_TLS = true +SKIP_TLS_VERIFY = true +REPLY_TO_ADDRESS = incoming+%{token}@localhost