diff --git a/synapse/cmds/cron.py b/synapse/cmds/cron.py index 25590c061bd..5d3587e8d93 100644 --- a/synapse/cmds/cron.py +++ b/synapse/cmds/cron.py @@ -454,8 +454,7 @@ async def _handle_add(self, core, opts): @staticmethod def _format_timestamp(ts): - # N.B. normally better to use fromtimestamp with UTC timezone, but we don't want timezone to print out - return datetime.datetime.utcfromtimestamp(ts).isoformat(timespec='minutes') + return datetime.datetime.fromtimestamp(ts, datetime.UTC).strftime('%Y-%m-%dT%H:%M') async def _handle_list(self, core, opts): cronlist = await core.listCronJobs() diff --git a/synapse/lib/stormlib/stix.py b/synapse/lib/stormlib/stix.py index ad3cf15e132..8d5f0640411 100644 --- a/synapse/lib/stormlib/stix.py +++ b/synapse/lib/stormlib/stix.py @@ -1144,8 +1144,9 @@ async def bundle(self, config=None): return StixBundle(self, self.runt, config) def timestamp(self, tick): - dt = datetime.datetime.utcfromtimestamp(tick / 1000.0) - return dt.isoformat(timespec='milliseconds') + 'Z' + dt = datetime.datetime.fromtimestamp(tick / 1000.0, datetime.UTC) + millis = int(dt.microsecond / 1000) + return f'{dt.strftime("%Y-%m-%dT%H:%M:%S")}.{millis:03d}Z' stix_sdos = { 'attack-pattern', diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 093d75e4000..a930a8ed0e5 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -9481,9 +9481,7 @@ async def _methCronJobPack(self): @staticmethod def _formatTimestamp(ts): - # N.B. normally better to use fromtimestamp with UTC timezone, - # but we don't want timezone to print out - return datetime.datetime.utcfromtimestamp(ts).isoformat(timespec='minutes') + return datetime.datetime.fromtimestamp(ts, datetime.UTC).strftime('%Y-%m-%dT%H:%M') async def _methCronJobPprint(self): user = self.valu.get('username') diff --git a/synapse/tests/test_common.py b/synapse/tests/test_common.py index 4aa0d2219f9..bf401bcc5bb 100644 --- a/synapse/tests/test_common.py +++ b/synapse/tests/test_common.py @@ -183,11 +183,11 @@ def test_common_chunks(self): parts = [chunk for chunk in s_common.chunks({1, 2, 3}, 10000)] # dict is unslicable - with self.assertRaises(TypeError) as cm: + with self.assertRaises((TypeError, KeyError)) as cm: parts = [chunk for chunk in s_common.chunks({1: 2}, 10000)] # empty dict is caught during the [0:0] slice - with self.assertRaises(TypeError) as cm: + with self.assertRaises((TypeError, KeyError)) as cm: parts = [chunk for chunk in s_common.chunks({}, 10000)] def test_common_ehex_uhex(self): diff --git a/synapse/tests/test_lib_link.py b/synapse/tests/test_lib_link.py index dea119bf34d..c93acffa172 100644 --- a/synapse/tests/test_lib_link.py +++ b/synapse/tests/test_lib_link.py @@ -1,7 +1,10 @@ import ssl +import sys import socket import asyncio +import multiprocessing +import synapse.exc as s_exc import synapse.common as s_common import synapse.lib.coro as s_coro @@ -9,6 +12,36 @@ import synapse.tests.utils as s_test + +# Helpers related to spawn link coverage +async def _spawnTarget(n, info): + link = await s_link.fromspawn(info) + async with link: + await link.send(b'V' * n) + +def spawnTarget(n, info): + asyncio.run(_spawnTarget(n, info)) + +async def _spawnHost(n, pipe): + link0, sock0 = await s_link.linksock() + info = await link0.getSpawnInfo() + pipe.send(info) + data = sock0.recv(n) + + sock0.close() + await link0.fini() + + if data == b'V' * n: + return + + return 137 + +def spawnHost(n, pipe: multiprocessing.Pipe): + ret = asyncio.run(_spawnHost(n, pipe)) + if ret is None: + return + sys.exit(ret) + class LinkTest(s_test.SynTest): async def test_link_raw(self): @@ -154,20 +187,61 @@ def writer(sock): await link1.fini() sock1.close() - async def test_link_fromspawn(self): + async def test_link_fromspawns(self): + + ctx = multiprocessing.get_context('spawn') + + # Remote use test - this is normally how linksock is used. link0, sock0 = await s_link.linksock() info = await link0.getSpawnInfo() - link1 = await s_link.fromspawn(info) - await link1.send(b'V') - self.eq(sock0.recv(1), b'V') + n = 100000 + def getproc(): + proc = ctx.Process(target=spawnTarget, args=(n, info)) + proc.start() + return proc - sock0.close() + proc = await s_coro.executor(getproc) + + self.eq(sock0.recv(n), b'V' * n) + await s_coro.executor(proc.join) + + sock0.close() await link0.fini() - await link1.fini() + + # Coverage test + mypipe, child_pipe = ctx.Pipe() + + def getproc(): + proc = ctx.Process(target=spawnHost, args=(n, child_pipe)) + proc.start() + return proc + + proc = await s_coro.executor(getproc) # type: multiprocessing.Process + + def waitforinfo(): + nonlocal proc + hasdata = mypipe.poll(timeout=30) + if not hasdata: + raise s_exc.SynErr(mesg='failed to get link info') + info = mypipe.recv() + return info + + info = await s_coro.executor(waitforinfo) + + link = await s_link.fromspawn(info) + await link.send(b'V' * n) + + def waitforjoin(): + proc.join() + return proc.exitcode + + code = await s_coro.executor(waitforjoin) + self.eq(code, 0) + await link.fini() async def test_tls_ciphers(self): self.thisHostMustNot(platform='darwin') diff --git a/synapse/tests/test_lib_multislabseqn.py b/synapse/tests/test_lib_multislabseqn.py index 9803b140885..16c1e1a5de3 100644 --- a/synapse/tests/test_lib_multislabseqn.py +++ b/synapse/tests/test_lib_multislabseqn.py @@ -13,7 +13,7 @@ class MultiSlabSeqn(s_t_utils.SynTest): - async def test_multislabseqn(self): + async def test_multislabseqn_base(self): with self.getTestDir() as dirn: @@ -166,10 +166,6 @@ async def getter(): exp = [(11, 'foo11b'), (12, 'foo12'), (13, 'foo13b'), (14, 'foo14'), (15, 'done')] self.eq(exp, retn) - # Give a chance for the non-iterated async generators to get cleaned up - await asyncio.sleep(0) - await asyncio.sleep(0) - # Make sure we're not holding onto more than 2 slabs # rotate @@ -181,6 +177,10 @@ async def getter(): fns = sorted(s_common.listdir(dirn, glob='*.lmdb')) self.len(3, fns) + # Slab @ 13 will get fini out + _slab = msqn._openslabs[13][0] + self.true(await _slab.waitfini(6)) + self.len(2, msqn._openslabs) retn = await msqn.get(11) # first diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index 9a4f858140d..0c7b3b15d74 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -2966,11 +2966,11 @@ async def test_storm_splicelist(self): mesgs = await core.stormlist('[ test:str=bar ]') tick = mesgs[0][1]['tick'] - tickdt = datetime.datetime.utcfromtimestamp(tick / 1000.0) + tickdt = datetime.datetime.fromtimestamp(tick / 1000.0, datetime.UTC) tickstr = tickdt.strftime('%Y/%m/%d %H:%M:%S.%f') tock = mesgs[-1][1]['tock'] - tockdt = datetime.datetime.utcfromtimestamp(tock / 1000.0) + tockdt = datetime.datetime.fromtimestamp(tock / 1000.0, datetime.UTC) tockstr = tockdt.strftime('%Y/%m/%d %H:%M:%S.%f') await asyncio.sleep(0.01)