1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# This webhook receives events from groups_io and injects them into the
# local mail queue to be processed as if it came in via the SMTP gateway.
#
__author__ = 'Konstantin Ryabitsev <konstantin@linuxfoundation.org>'
import falcon # noqa
import json
import socket
import email
import email.utils
import email.header
import email.policy
import hmac
import smtplib
import os
import sys
import logging
import logging.handlers
import datetime
from configparser import ConfigParser, ExtendedInterpolation
logger = logging.getLogger('groupsio-webhook')
logger.setLevel(logging.DEBUG)
emlpolicy = email.policy.EmailPolicy(utf8=True, cte_type='8bit', max_line_length=None)
# noinspection PyBroadException, PyMethodMayBeStatic
class GroupsioListener(object):
_config: ConfigParser
psk: str
def __init__(self, _config: ConfigParser) -> None:
self._config = _config
self.psk = _config['main'].get('groupsio_psk')
logfile = _config['main'].get('logfile')
loglevel = _config['main'].get('loglevel', 'info')
if logfile:
self._init_logger(logfile, loglevel)
def _init_logger(self, logfile: str, loglevel: str) -> None:
global logger
lch = logging.handlers.WatchedFileHandler(os.path.expanduser(logfile))
lfmt = logging.Formatter('[%(process)d] %(asctime)s - %(levelname)s - %(message)s')
lch.setFormatter(lfmt)
if loglevel == 'critical':
lch.setLevel(logging.CRITICAL)
elif loglevel == 'debug':
lch.setLevel(logging.DEBUG)
else:
lch.setLevel(logging.INFO)
logger.addHandler(lch)
def on_get(self, req, resp): # noqa
resp.status = falcon.HTTP_200
resp.body = "We don't serve GETs here\n"
def _inject_message(self, doc: dict, req) -> bool:
success = False
msgnum = doc['msg_num']
groupaddr = doc['group']['email_address']
groupurl = doc['group']['group_url']
logger.info('Received %s msg %s', groupaddr, msgnum)
logger.debug('---%s START---', msgnum)
logger.debug(doc['message'])
logger.debug('---%s END---', msgnum)
try:
msg = email.message_from_string(doc['message'])
try:
theirname = socket.gethostbyaddr(req.remote_addr)[0]
them = f'{theirname} [{req.remote_addr}]'
except:
them = f'[{req.remote_addr}]'
try:
us = socket.gethostname()
except:
us = 'localhost'
logger.debug('us=%s', us)
logger.debug('them=%s', them)
listid = groupaddr.replace('@', '.')
scheme = req.scheme.upper()
rdate = email.utils.formatdate()
rline = f'from {them} by {us} with {scheme} for <{groupaddr}>; {rdate}'
rhdr = email.header.make_header([(rline.encode(), 'us-ascii')], maxlinelen=78)
if msg.get('list-id'):
msg.replace_header('List-Id', f'<{listid}>')
else:
msg.add_header('List-Id', f'<{listid}>')
msg.add_header('X-Webhook-Received', rhdr.encode())
msg.add_header('X-Groupsio-URL', f'{groupurl}/message/{msgnum}')
# do we have a repodir for this list?
if groupaddr in self._config.sections():
pirepo = self._config[groupaddr].get('pirepo')
success = self._write_pi(pirepo, msg)
if not success:
try:
mailfrom = doc['member_info']['email']
except:
mailfrom = 'webhook@localhost'
success = self._send_smtp(msg, mailfrom)
except Exception as ex:
logger.critical('Failed processing message %s:', msgnum)
logger.critical(str(ex))
filen = f'{groupaddr}-{msgnum}.txt'
self._save_emerg(filen, doc['message'].encode())
return success
def _write_pi(self, pirepo: str, msg: email.message.Message) -> bool:
success = False
try:
import ezpi # noqa
ezpi.add_rfc822(pirepo, msg)
ezpi.run_hook(pirepo)
logger.info('Wrote to public-inbox at %s', pirepo)
success = True
except Exception as ex:
logger.critical('Could not write to pirepo')
logger.critical(str(ex))
return success
def _send_smtp(self, msg: email.message.Message, mailfrom: str) -> bool:
mailhost = self._config['main'].get('mailhost', 'localhost')
mailto = self._config['main'].get('mailto', 'webhook@archiver.kernel.org')
bdata = msg.as_string(policy=emlpolicy).encode()
try:
smtp = smtplib.SMTP(mailhost)
smtp.sendmail(mailfrom, [mailto], bdata)
smtp.close()
logger.info('Successfully sent via SMTP to <%s>', mailto)
except Exception as ex:
logger.critical('Could not send SMTP')
logger.critical(str(ex))
dt = datetime.datetime.now()
fname = dt.strftime('%Y%m%d-%H%M%S-%f') + '.eml'
self._save_emerg(fname, bdata)
return False
return True
def _verify_psk(self, psk, raw, vdigest):
hm = hmac.new(psk.encode(), digestmod='sha256')
hm.update(raw)
return hmac.compare_digest(vdigest, hm.hexdigest())
def _save_emerg(self, filen: str, bdata: bytes):
emerg_dir = self._config['main'].get('emerg_dir')
if not (emerg_dir or os.path.isdir(emerg_dir)):
return
efile = os.path.join(emerg_dir, filen)
with open(efile, 'wb') as fh:
fh.write(bdata)
logger.debug('Wrote emergency content into %s', efile)
def on_post(self, req, resp):
if not req.content_length:
resp.status = falcon.HTTP_500
resp.body = 'Payload required\n'
return
logger.info('Received POST from %s', req.remote_addr)
raw = req.stream.read()
if self.psk:
vdigest = req.get_header('X-Groupsio-Signature')
if not vdigest:
resp.status = falcon.HTTP_401
resp.body = 'HMAC signature header required\n'
return
if not self._verify_psk(self.psk, raw, vdigest):
resp.status = falcon.HTTP_401
resp.body = 'HMAC signature verification failed\n'
return
if self._config['main'].get('loglevel', 'info') == 'debug':
# In debug mode, we save all json submissions in the emerg dir
dt = datetime.datetime.now()
fname = dt.strftime('%Y%m%d-%H%M%S-%f') + '.js'
self._save_emerg(fname, raw)
try:
doc = json.loads(raw)
except Exception as ex:
resp.status = falcon.HTTP_500
resp.body = 'Failed to parse payload as json\n'
logger.critical('Failed to parse incoming json:')
logger.critical(str(ex))
return
success = True
if 'message' in doc and len(doc.get('message')):
success = self._inject_message(doc, req)
else:
logger.debug('No message in payload, skipping')
if success:
resp.status = falcon.HTTP_200
resp.body = 'OK thanks\n'
else:
resp.status = falcon.HTTP_500
resp.body = 'Something went wrong, sorry.\n'
parser = ConfigParser(interpolation=ExtendedInterpolation())
cfgfile = os.getenv('CONFIG')
if not cfgfile or not os.path.exists(cfgfile):
sys.stderr.write('CONFIG env var is not set or is not valid')
sys.exit(1)
parser.read(cfgfile)
gl = GroupsioListener(parser)
app = falcon.App()
mp = os.getenv('MOUNTPOINT', '/groupsio_webhook')
app.add_route(mp, gl)
|
