Repository navigation
Expand file tree
/
Copy pathserver.py
More file actions
executable file
·237 lines (198 loc) · 10.7 KB
/
Copy pathserver.py
File metadata and controls
executable file
·237 lines (198 loc) · 10.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
#!/usr/bin/env python
"""
Copyright (c) 2014-present Maltrail developers (https://github.com/stamparm/maltrail/)
See the file 'LICENSE' for copying permission
"""
import sys
sys.dont_write_bytecode = True
import argparse
import errno
import os
import re
import platform
import threading
import time
import traceback
from core import alert
from core.common import check_connection
from core.common import check_sudo
from core.common import get_ex_message
from core.common import patch_parser
from core.common import uses_published_key
from core.httpd import is_published_password
from core.httpd import start_httpd
from core import password
from core import meta
from core.log import create_log_directory
from core.log import log_error
from core.log import start_logd
from core.settings import config
from core.settings import read_config
from core.settings import CHECK_CONNECTION_MAX_RETRIES
from core.settings import CONFIG_FILE
from core.settings import HOMEPAGE
from core.settings import IS_WIN
from core.settings import META_DB_FILENAME
from core.settings import NAME
from core.settings import VERSION
from core.update import update_geo
from core.update import update_drop
from core.update import update_ipcat
from core.update import fetch_provenance
from core.update import update_trails
def main():
print("%s (server) #v%s {%s}\n" % (NAME, VERSION, HOMEPAGE))
if "--version" in sys.argv:
raise SystemExit
parser = argparse.ArgumentParser()
parser.add_argument("--version", action="version", version=VERSION)
parser.add_argument("-c", dest="config_file", default=CONFIG_FILE, help="configuration file (default: '%s')" % os.path.split(CONFIG_FILE)[-1])
parser.add_argument("--debug", dest="debug", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--smoke-test", dest="smoke_test", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--detect-test", dest="detect_test", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--keep", dest="keep", default=None, metavar="DIR", help="with --detect-test: keep the generated events in DIR (and replay the parity corpus into it) instead of deleting them")
parser.add_argument("--serve", dest="serve", action="store_true", help="with --detect-test --keep: start the web server on the generated events")
parser.add_argument("--hash-password", dest="hash_password", action="store_true", help="read a password and print the USERS field for it")
parser.add_argument("--doctor", dest="doctor", action="store_true", help="validate the deployment (log dir, trails age, USERS, TLS, ports) and exit")
patch_parser(parser)
options = parser.parse_args()
if options.hash_password:
from core.password import main as hash_password
raise SystemExit(hash_password())
if options.smoke_test:
from core.testing import smoke_test
raise SystemExit(0 if smoke_test() else 1)
if options.detect_test:
from core.testing import detect_test
raise SystemExit(0 if detect_test(keep=options.keep, serve=options.serve) else 1)
if options.serve or options.keep:
sys.exit("[!] '--keep' and '--serve' only apply together with '--detect-test'")
print("[*] starting @ %s\n" % time.strftime("%X /%Y-%m-%d/"))
read_config(options.config_file)
if options.debug:
config.SHOW_DEBUG = True
if options.doctor:
from core.doctor import run as doctor_run
raise SystemExit(doctor_run())
# The per-day SQLite sidecars of the retired event index (LOG_DIR/index/) were a cache of
# ours, about twice the size of the logs they indexed; nothing reads them any more.
# Only the files it wrote (YYYY-MM-DD.sqlite and its WAL/SHM companions) are removed.
stale_index = os.path.join(config.LOG_DIR or "", "index")
if config.LOG_DIR and os.path.isdir(stale_index):
removed = 0
for name in os.listdir(stale_index):
if re.match(r"\A\d{4}-\d{2}-\d{2}\.sqlite(-wal|-shm|-journal)?\Z", name):
try:
os.remove(os.path.join(stale_index, name))
removed += 1
except OSError:
pass
try:
os.rmdir(stale_index)
except OSError:
pass
if removed:
print("[i] removed %d file(s) of the retired event index in '%s'" % (removed, stale_index))
# NOTE: this validation used to live inside an `if six.PY2 and config.USE_SSL:` block, so on
# Python 3 it never ran and a missing/invalid SSL_PEM only failed later, inside the server
# thread. It is a plain USE_SSL check now.
if config.USE_SSL:
hint = "openssl req -new -x509 -keyout %s -out %s -days 365 -nodes -subj '/O=%s CA/C=EU'" % (config.SSL_PEM or "server.pem", config.SSL_PEM or "server.pem", NAME)
if not config.SSL_PEM or not os.path.isfile(config.SSL_PEM):
sys.exit("[!] invalid configuration value for 'SSL_PEM' ('%s')\n[?] (hint: \"%s\")" % (config.SSL_PEM, hint))
# The key Maltrail used to ship in misc/server.pem is public - it sat in a public
# repository from 2020 and is still in its git history - so TLS with it protects nothing.
# Anyone who copied it once keeps serving HTTPS that anybody can impersonate or decrypt,
# and nothing tells them: the padlock looks identical. Refuse instead of pretending.
if uses_published_key(config.SSL_PEM):
sys.exit("[!] 'SSL_PEM' ('%s') is the key %s published in misc/server.pem, which is public and provides no protection\n[?] (hint: \"%s\")" % (config.SSL_PEM, NAME, hint))
users = [_.split(':', 3) for _ in (re.sub(r"\s", "", _) for _ in (config.USERS or []))]
legacy = [_[0] for _ in users if len(_) == 4 and not password.parse(_[1])]
if legacy:
print("[!] 'USERS' stores sha256(password) for %s: that value logs in as well as the password does, for anyone who can read the configuration. Regenerate it with 'python3 server.py --hash-password'" % ", ".join("'%s'" % _ for _ in legacy))
if any(len(_) == 4 and is_published_password(_[1]) for _ in users):
print("[!] 'USERS' still holds the published default password ('changeme!'): it is accepted only from this machine, set your own to sign in over the network")
def update_timer():
retries = 0
while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection():
sys.stdout.write("[!] can't update because of lack of Internet connection (waiting..." if not retries else '.')
sys.stdout.flush()
time.sleep(10)
retries += 1
if retries:
print(")")
if retries == CHECK_CONNECTION_MAX_RETRIES:
print("[x] going to continue without online update")
_ = update_trails(offline=True)
else:
_ = update_trails()
fetch_provenance()
update_ipcat()
update_geo()
update_drop()
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
if config.UDP_ADDRESS and config.UDP_PORT:
if config.UDP_PORT <= 1024 and not config.DISABLE_CHECK_SUDO and check_sudo() is False:
sys.exit("[!] please run '%s' with root privileges when using 'UDP_ADDRESS' configuration value" % __file__)
create_log_directory()
start_logd(address=config.UDP_ADDRESS, port=config.UDP_PORT, join=False)
# configure the condensed observable store for read-side /meta lookups + prune (the sensor writes it)
meta.configure(os.path.join(config.LOG_DIR, META_DB_FILENAME), enabled=False)
alert.start() # no-op unless ALERT_WEBHOOK_URL is set
try:
if config.USE_SERVER_UPDATE_TRAILS:
# In the background: run first, it kept the dashboard (and its healthcheck) down for
# the whole update - every feed on a first boot or after an outage, and up to three
# 40 s connectivity checks offline. The update re-arms its own timer when it is done.
thread = threading.Thread(target=update_timer)
thread.daemon = True
thread.start()
start_httpd(address=config.HTTP_ADDRESS, port=config.HTTP_PORT, pem=config.SSL_PEM if config.USE_SSL else None, join=True)
except KeyboardInterrupt:
print("\r[x] stopping (Ctrl-C pressed)")
if __name__ == "__main__":
code = 0
try:
main()
except SystemExit as ex:
if isinstance(get_ex_message(ex), str) and get_ex_message(ex).strip('0'):
print(get_ex_message(ex))
code = 1
# NOTE: this used to be a bare `except IOError`, and IOError is an alias of OSError - so EVERY OSError
# raised anywhere in main() was reported as "session abruptly terminated", which log_error() writes to
# the log file and never prints. `server.py --smoke-test` on a tree with a root-owned __pycache__ was
# the demonstration: PermissionError from py_compile, no output whatsoever, exit 1. A tool that fails
# silently sends the operator looking in the wrong place. Only a broken pipe is handled quietly, and
# only because there is nowhere left to print to.
except OSError as ex:
if getattr(ex, "errno", None) in (errno.EPIPE, errno.ESHUTDOWN):
log_error("\n\n[!] session abruptly terminated\n[?] (hint: \"https://stackoverflow.com/a/20997655\")")
code = 1
else:
msg = "\r[!] unhandled exception occurred ('%s')" % ex
msg += "\n[x] please report the following details at 'https://github.com/stamparm/maltrail/issues':\n---\n'%s'\n---" % traceback.format_exc()
log_error("\n\n%s" % msg.replace("\r", ""))
print(msg)
code = 1
except Exception:
msg = "\r[!] unhandled exception occurred ('%s')" % sys.exc_info()[1]
msg += "\n[x] please report the following details at 'https://github.com/stamparm/maltrail/issues':\n---\n'%s'\n---" % traceback.format_exc()
log_error("\n\n%s" % msg.replace("\r", ""))
print(msg)
code = 1
finally:
if not any(_ in sys.argv for _ in ("--version", "-h", "--help")):
print("\n[*] ending @ %s" % time.strftime("%X /%Y-%m-%d/"))
# os._exit() skips interpreter cleanup, which includes flushing stdout/stderr. When
# stdout is not a terminal it is block-buffered, so without this every line printed
# during the run is silently discarded (`server.py --version | cat` printed nothing).
# _exit() itself is deliberate: a worker parked in a blocking read can hold the stdio
# lock, and a normal exit() would block forever trying to flush it.
try:
sys.stdout.flush()
sys.stderr.flush()
except Exception:
pass
os._exit(code)