From e7da42b20434552dbff1a496a38eb333be75cead Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Mon, 27 Jul 2015 01:16:09 -0700 Subject: [PATCH] watchman: start building python based test suite Summary: We've had a number of compatibility issues with the arcanist based test suite, so this is an effort to run them using the python unittest infrastructure. This new structure allows us to create a single temporary dir and to create a dir per test case to track the temporary files and dirs created during the test. This is beneficial both from a post-mortem perspective if a test fails, but also because the paths that show up in the watchman logs will now be easily recognizable as being associated with a given test. This will also help us manage the windows integration tests (https://github.com/facebook/watchman/issues/19#issuecomment-125054948) a bit more sanely; a source of errors in the php tests is that deleting directory trees can fail if a handle still references any part of it, and there is often a noticeable lag where we can hit this state and error out. By deferring the deletes until our process end, we should minimize this issue. I've ported a single integration test to demonstrate what this looks like, the rest will have to be a series of diffs for easier review. Test Plan: `make integration` or `./runtests.py` Reviewers: sid0 Differential Revision: https://reviews.facebook.net/D43137 --- Makefile.am | 8 +- python/tests/__init__.py | 0 runtests.py | 100 ++++++++++++ tests/integration/WatchmanInstance.py | 68 ++++++++ tests/integration/WatchmanTestCase.py | 112 +++++++++++++ tests/integration/__init__.py | 0 tests/integration/test_since.py | 222 ++++++++++++++++++++++++++ 7 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 python/tests/__init__.py create mode 100755 runtests.py create mode 100644 tests/integration/WatchmanInstance.py create mode 100644 tests/integration/WatchmanTestCase.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_since.py diff --git a/Makefile.am b/Makefile.am index 54d1c7f612de..d6cb8b14579a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,7 +108,10 @@ py-install: # This is invoked via WatchmanIntegrationEngine py-tests: - time $(PYTHON) $(TESTNAME) + $(PYTHON) $(TESTNAME) + +py-integration: + $(PYTHON) runtests.py py-clean: -cd python && $(PYTHON) ./setup.py clean --all @@ -118,6 +121,7 @@ py-build: py-tests: py-clean: py-install: +py-integration: endif if HAVE_RUBY @@ -150,7 +154,7 @@ clean-local: py-clean rb-clean build-tests: $(TESTS) .PHONY: lint build-tests integration py-tests # run integration AND unit tests -integration: +integration: py-integration arc test tests_argv_t_CPPFLAGS = $(THIRDPARTY_CPPFLAGS) diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/runtests.py b/runtests.py new file mode 100755 index 000000000000..98cb93434742 --- /dev/null +++ b/runtests.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# vim:ts=4:sw=4:et: +import unittest +import os +import os.path +import sys +import tempfile +import shutil +import subprocess +import traceback +import time +import argparse +import atexit + +parser = argparse.ArgumentParser( + description="Run the watchman unit and integration tests") +parser.add_argument('-v', '--verbosity', default=2, + help="test runner verbosity") +parser.add_argument( + "--keep", + action='store_true', + help="preserve all temporary files created during test execution") +args = parser.parse_args() + +# Ensure that we can find pywatchman +sys.path.append(os.path.join(os.getcwd(), 'python')) +sys.path.append(os.path.join(os.getcwd(), 'tests/integration')) + +# We test for this in a test case +os.environ['WATCHMAN_EMPTY_ENV_VAR'] = '' + +unittest.installHandler() + +# We'll put all our temporary stuff under one dir so that we +# can clean it all up at the end +temp_dir = tempfile.mkdtemp(prefix='watchmantest') +if args.keep: + atexit.register(sys.stdout.write, + 'Preserving output in %s\n' % temp_dir) +else: + atexit.register(shutil.rmtree, temp_dir) +# Redirect all temporary files to that location +tempfile.tempdir = temp_dir + +# Start up a shared watchman instance for the tests. +# We defer the import until after we've modified the python path +import WatchmanInstance +inst = WatchmanInstance.Instance() +inst.start() + +# Allow tests to locate our instance by default +os.environ['WATCHMAN_SOCK'] = inst.getSockPath() + + +class Result(unittest.TestResult): + # Make it easier to spot success/failure by coloring the status + # green for pass, red for fail and yellow for skip. + # also print the elapsed time per test + + def startTest(self, test): + self.startTime = time.time() + super(Result, self).startTest(test) + + def addSuccess(self, test): + elapsed = time.time() - self.startTime + super(Result, self).addSuccess(test) + print('\033[32mPASS\033[0m %s (%.3fs)' % (test.id(), elapsed)) + + def addSkip(self, test, reason): + elapsed = time.time() - self.startTime + super(Result, self).addSkip(test, reason) + print('\033[33mSKIP\033[0m %s (%.3fs) %s' % + (test.id(), elapsed, reason)) + + def __printFail(self, test, err): + elapsed = time.time() - self.startTime + t, val, trace = err + print('\033[31mFAIL\033[0m %s (%.3fs)\n%s' % ( + test.id(), + elapsed, + ''.join(traceback.format_exception(t, val, trace)))) + + def addFailure(self, test, err): + self.__printFail(test, err) + super(Result, self).addFailure(test, err) + + def addError(self, test, err): + self.__printFail(test, err) + super(Result, self).addError(test, err) + + +loader = unittest.TestLoader() +suite = unittest.TestSuite() +for d in ['python/tests', 'tests/integration']: + suite.addTests(loader.discover(d, top_level_dir=d)) + +unittest.TextTestRunner( + resultclass=Result, + verbosity=args.verbosity +).run(suite) diff --git a/tests/integration/WatchmanInstance.py b/tests/integration/WatchmanInstance.py new file mode 100644 index 000000000000..f5843b84c658 --- /dev/null +++ b/tests/integration/WatchmanInstance.py @@ -0,0 +1,68 @@ +# vim:ts=4:sw=4:et: +# Copyright 2012-present Facebook, Inc. +# Licensed under the Apache License, Version 2.0 +import tempfile +import json +import os.path +import subprocess +import pywatchman +import time + + +class Instance(object): + # Tracks a running watchman instance. It is created with an + # overridden global configuration file; you may pass that + # in to the constructor + + def __init__(self, config={}): + self.base_dir = tempfile.mkdtemp(prefix='inst') + self.cfg_file = os.path.join(self.base_dir, "config.json") + self.log_file_name = os.path.join(self.base_dir, "log") + self.sock_file = os.path.join(self.base_dir, "sock") + self.state_file = os.path.join(self.base_dir, "state") + with open(self.cfg_file, "w") as f: + f.write(json.dumps(config)) + self.log_file = open(self.log_file_name, 'w+') + + def __del__(self): + self.stop() + + def getSockPath(self): + return self.sock_file + + def stop(self): + if self.proc: + self.proc.kill() + self.proc.wait() + self.log_file.close() + + def start(self): + args = [ + './watchman', + '--foreground', + '--sockname={}'.format(self.sock_file), + '--logfile={}'.format(self.log_file_name), + '--statefile={}'.format(self.state_file), + '--log-level=2', + ] + env = os.environ.copy() + env["WATCHMAN_CONFIG_FILE"] = self.cfg_file + self.proc = subprocess.Popen(args, + env=env, + stdin=None, + stdout=self.log_file, + stderr=self.log_file) + + # wait for it to come up + last_err = None + for i in xrange(1, 10): + try: + client = pywatchman.client(sockpath=self.sock_file) + self.pid = client.query('get-pid')['pid'] + break + except Exception as e: + last_err = e + time.sleep(0.1) + + if not self.pid: + raise last_err diff --git a/tests/integration/WatchmanTestCase.py b/tests/integration/WatchmanTestCase.py new file mode 100644 index 000000000000..ec237b593361 --- /dev/null +++ b/tests/integration/WatchmanTestCase.py @@ -0,0 +1,112 @@ +# vim:ts=4:sw=4:et: +# Copyright 2012-present Facebook, Inc. +# Licensed under the Apache License, Version 2.0 +import errno +import unittest +import pywatchman +import time +import tempfile +import os.path +import os + + +class WatchmanTestCase(unittest.TestCase): + + def getClient(self): + if not hasattr(self, 'client'): + self.client = pywatchman.client() + return self.client + + def __logTestInfo(self, test, msg): + if hasattr(self, 'client'): + try: + self.getClient().query('log', 'debug', + 'TEST: %s %s\n\n' % (test, msg)) + except Exception as e: + pass + + def run(self, result=None): + # Arrange for any temporary stuff we create to go under + # our global tempdir and put it in a dir named for the test + saved_root = tempfile.tempdir + try: + tempfile.tempdir = os.path.join(saved_root, self.id()) + os.mkdir(tempfile.tempdir) + self.__logTestInfo(self.id(), 'BEGIN') + return super(WatchmanTestCase, self).run(result) + finally: + tempfile.tempdir = saved_root + self.__logTestInfo(self.id(), 'END') + + def touch(self, fname, times=None): + try: + os.utime(fname, times) + except OSError as e: + if e.errno == errno.ENOENT: + with open(fname, 'a'): + os.utime(fname, times) + else: + raise + + def touchRelative(self, base, *fname): + fname = os.path.join(base, *fname) + self.touch(fname, None) + + def __del__(self): + if hasattr(self, 'client'): + try: + self.watchmanCommand('watch-del-all') + except Exception as e: + pass + + def watchmanCommand(self, *args): + return self.getClient().query(*args) + + # Continually invoke `cond` until it returns true or timeout + # is reached. Returns a tuple of [bool, result] where the + # first element of the tuple indicates success/failure and + # the second element is the return value from the condition + def waitFor(self, cond, timeout=10): + deadline = time.time() + timeout + res = None + while time.time() < deadline: + res = cond() + if res: + return [True, res] + time.sleep(0.03) + return [False, res] + + def assertWaitFor(self, cond, timeout=10, message=None): + status, res = self.waitFor(cond, timeout) + if status: + return res + if message is None: + message = "%s was not met in %s seconds: %s" % (cond, timeout, res) + self.fail(message) + + def getFileList(self, root, cursor=None, relativeRoot=None): + expr = { + "expression": ["exists"], + "fields": ["name"], + } + if cursor: + expr['since'] = cursor + if relativeRoot: + expr['relative_root'] = relativeRoot + res = self.watchmanCommand('query', root, expr) + files = sorted(res['files']) + self.last_file_list = files + return files + + def normFileList(self, files): + return sorted(map(os.path.normpath, files)) + + # Wait for the file list to match the input set + def assertFileList(self, root, files=[], cursor=None, + relativeRoot=None, message=None): + expected_files = self.normFileList(files) + st, res = self.waitFor( + lambda: self.getFileList(root, cursor=cursor, + relativeRoot=relativeRoot + ) == expected_files) + self.assertEqual(self.last_file_list, expected_files, message) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_since.py b/tests/integration/test_since.py new file mode 100644 index 000000000000..3f06da60c331 --- /dev/null +++ b/tests/integration/test_since.py @@ -0,0 +1,222 @@ +# vim:ts=4:sw=4:et: +# Copyright 2012-present Facebook, Inc. +# Licensed under the Apache License, Version 2.0 +import WatchmanTestCase +import tempfile +import os +import os.path + + +class TestSince(WatchmanTestCase.WatchmanTestCase): + + def test_sinceIssue1(self): + root = tempfile.mkdtemp() + self.touchRelative(root, '111') + self.touchRelative(root, '222') + + self.watchmanCommand('watch', root) + self.assertFileList(root, ['111', '222']) + + # Create a cursor for this state + self.watchmanCommand('since', root, 'n:foo') + + bar_dir = os.path.join(root, 'bar') + os.mkdir(bar_dir) + self.touchRelative(bar_dir, '333') + + # We should not observe 111 or 222 + self.assertFileList(root, cursor='n:foo', files=['bar', 'bar/333']) + + def test_sinceIssue2(self): + root = tempfile.mkdtemp() + self.watchmanCommand('watch', root) + self.assertFileList(root, files=[]) + + foo_dir = os.path.join(root, 'foo') + os.mkdir(foo_dir) + self.touchRelative(foo_dir, '111') + + self.assertFileList(root, cursor='n:foo', files=['foo', 'foo/111']) + + bar_dir = os.path.join(foo_dir, 'bar') + os.mkdir(bar_dir) + self.touchRelative(bar_dir, '222') + + # wait until we observe all the files + self.assertFileList(root, files=[ + 'foo', + 'foo/111', + 'foo/bar', + 'foo/bar/222']) + + # now check the delta for the since + self.assertFileList(root, cursor='n:foo', files=[ + 'foo/bar', + 'foo/bar/222']) + + def test_sinceRelativeRoot(self): + root = tempfile.mkdtemp() + self.watchmanCommand('watch', root) + clock = self.watchmanCommand('clock', root)['clock'] + + self.touchRelative(root, 'a') + os.mkdir(os.path.join(root, 'subdir')) + self.touchRelative(os.path.join(root, 'subdir'), 'foo') + self.assertFileList(root, files=[ + 'a', + 'subdir', + 'subdir/foo']) + + res = self.watchmanCommand('query', root, { + 'since': clock, + 'relative_root': 'subdir', + 'fields': ['name']}) + self.assertEqual(self.normFileList(res['files']), ['foo']) + + # touch a file outside the relative root + self.touchRelative(root, 'b') + res = self.watchmanCommand('query', root, { + 'since': res['clock'], + 'relative_root': 'subdir', + 'fields': ['name']}) + self.assertEqual(self.normFileList(res['files']), []) + + # touching just the subdir shouldn't cause anything to show up + self.touchRelative(root, 'subdir') + res = self.watchmanCommand('query', root, { + 'since': res['clock'], + 'relative_root': 'subdir', + 'fields': ['name']}) + self.assertEqual(self.normFileList(res['files']), []) + + # touching a new file inside the subdir should cause it to show up + dir2 = os.path.join(root, 'subdir', 'dir2') + os.mkdir(dir2) + self.touchRelative(dir2, 'bar') + res = self.watchmanCommand('query', root, { + 'since': res['clock'], + 'relative_root': 'subdir', + 'fields': ['name']}) + self.assertEqual(self.normFileList(res['files']), ['dir2', 'dir2/bar']) + + def assertFreshInstanceForSince(self, root, cursor, empty=False): + res = self.watchmanCommand('query', root, { + 'since': cursor, + 'fields': ['name'], + 'empty_on_fresh_instance': empty}) + self.assertTrue(res['is_fresh_instance']) + if empty: + self.assertEqual(res['files'], []) + else: + self.assertEqual(res['files'], ['111']) + + def test_sinceFreshInstance(self): + root = tempfile.mkdtemp() + self.watchmanCommand('watch', root) + self.assertFileList(root, []) + self.touchRelative(root, '111') + + res = self.watchmanCommand('query', root, { + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + self.assertEqual(res['files'], ['111']) + + # relative clock value, fresh instance + self.assertFreshInstanceForSince(root, 'c:0:1:0:1', False) + + # old-style clock value (implies fresh instance, event if the + # pid is the same) + pid = self.watchmanCommand('get-pid')['pid'] + self.assertFreshInstanceForSince(root, 'c:%s:1' % pid, False) + + # -- decompose clock and replace elements one by one + clock = self.watchmanCommand('clock', root)['clock'] + p = clock.split(':') + # ['c', startTime, pid, rootNum, ticks] + self.assertEqual(len(p), 5) + + # replace start time + self.assertFreshInstanceForSince(root, + ':'.join( + ['c', '0', p[2], p[3], p[4]]), + False) + + # replace pid + self.assertFreshInstanceForSince(root, + ':'.join( + ['c', p[1], '1', p[3], p[4]]), + False) + + # replace root number (also try empty_on_fresh_instance) + self.assertFreshInstanceForSince(root, + ':'.join( + ['c', p[1], p[2], '0', p[4]]), + True) + + # empty_on_fresh_instance, not a fresh instance + self.touchRelative(root, '222') + res = self.watchmanCommand('query', root, { + 'since': clock, + 'fields': ['name'], + 'empty_on_fresh_instance': True}) + self.assertFalse(res['is_fresh_instance']) + self.assertEqual(res['files'], ['222']) + + # fresh instance results should omit deleted files + os.unlink(os.path.join(root, '111')) + res = self.watchmanCommand('query', root, { + 'since': 'c:0:1:0:1', + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + self.assertEqual(self.normFileList(res['files']), ['222']) + + def test_reAddWatchFreshInstance(self): + root = tempfile.mkdtemp() + self.watchmanCommand('watch', root) + self.assertFileList(root, []) + self.touchRelative(root, '111') + + res = self.watchmanCommand('query', root, { + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + self.assertEqual(res['files'], ['111']) + + clock = res['clock'] + os.unlink(os.path.join(root, '111')) + self.watchmanCommand('watch-del', root) + + self.watchmanCommand('watch', root) + self.touchRelative(root, '222') + + # wait for touch to be observed + self.assertFileList(root, ['222']) + + # ensure that our since query is a fresh instance + res = self.watchmanCommand('query', root, { + 'since': clock, + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + self.assertEqual(res['files'], ['222']) + + def test_recrawlFreshInstance(self): + root = tempfile.mkdtemp() + self.watchmanCommand('watch', root) + self.touchRelative(root, '111') + self.assertFileList(root, ['111']) + + res = self.watchmanCommand('query', root, { + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + + clock = res['clock'] + os.unlink(os.path.join(root, '111')) + self.watchmanCommand('debug-recrawl', root) + + self.touchRelative(root, '222') + res = self.watchmanCommand('query', root, { + 'since': clock, + 'fields': ['name']}) + self.assertTrue(res['is_fresh_instance']) + self.assertEqual(res['files'], ['222']) + self.assertRegexpMatches(res['warning'], 'Recrawled this watch') +