diff --git a/src/runtime/python/README.md b/src/runtime/python/README.md
index 8878198d4..b11f2a7f1 100644
--- a/src/runtime/python/README.md
+++ b/src/runtime/python/README.md
@@ -21,5 +21,7 @@ See: https://www.grammaticalframework.org/doc/runtime-api.html#python
 ## Running tests
 
 ```sh
-python test.py
+pytest
 ```
+
+Requires: `pip install pytest`
diff --git a/src/runtime/python/test.py b/src/runtime/python/test.py
deleted file mode 100644
index 5f2549f00..000000000
--- a/src/runtime/python/test.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import pgf
-import sys
-# import sets
-# import readline
-# import locale
-
-sys.stdout.write("loading...")
-sys.stdout.flush();
-gr1 = pgf.readPGF("../haskell/tests/basic.pgf")
-gr2 = pgf.bootNGF("../haskell/tests/basic.pgf", "./basic.ngf")
-gr3 = pgf.readNGF("./basic.ngf")
-sys.stdout.write("done")
-
-# source_lang = gr.languages["ParseEng"]
-# target_lang = gr.languages["ParseBul"]
-#
-# we = pgf.readExpr("UttImpSg PPos (ImpVP (UseV try_V))")
-# print source_lang.linearize(we)
-#
-# sys.stdout.write("start cat: "+gr.startCat+"\n\n")
-#
-# class Completer():
-# 	def __init__(self, lang):
-# 		self.gr = lang
-#
-# 	def complete(self, prefix, state):
-# 		if state == 0:
-# 			line = readline.get_line_buffer()
-# 			line = line[0:readline.get_begidx()]
-# 			self.i = source_lang.complete(line, prefix=prefix)
-# 			self.tokens = sets.Set()
-#
-# 		if len(self.tokens) > 50:
-# 			return None
-#
-# 		while True:
-# 			try:
-# 				(p,t,c) = self.i.next()
-# 				if t not in self.tokens:
-# 					self.tokens.add(t)
-# 					return t
-# 			except StopIteration:
-# 				return None
-#
-# completer = Completer(source_lang)
-# readline.set_completer(completer.complete)
-# readline.parse_and_bind("tab: complete")
-# locale.setlocale(locale.LC_CTYPE, "")
-#
-# while True:
-# 	try:
-# 		line = raw_input("> ");
-# 	except EOFError:
-# 		sys.stdout.write("\n")
-# 		readline.set_completer(None)
-# 		break
-# 	except KeyboardInterrupt:
-# 		sys.stdout.write("\n")
-# 		readline.set_completer(None)
-# 		break
-#
-# 	try:
-# 		for (p,e) in source_lang.parse(line, n=1):
-# 			sys.stdout.write("["+str(p)+"] "+str(e)+"\n")
-# 			print target_lang.linearize(e)
-# 	except pgf.ParseError as e:
-# 		print e.message
diff --git a/src/runtime/python/test_suite.py b/src/runtime/python/test_suite.py
new file mode 100644
index 000000000..a10da14ff
--- /dev/null
+++ b/src/runtime/python/test_suite.py
@@ -0,0 +1,65 @@
+import pytest
+import pgf
+
+# readPGF
+
+@pytest.fixture(scope="module")
+def PGF():
+    return pgf.readPGF("../haskell/tests/basic.pgf")
+
+def test_readPGF_non_existant():
+    with pytest.raises(FileNotFoundError):
+        pgf.readPGF("../haskell/tests/abc.pgf")
+
+def test_readPGF_GF():
+    with pytest.raises(pgf.PGFError):
+        pgf.readPGF("../haskell/tests/basic.gf")
+
+@pytest.mark.skip(reason="Bug in runtime")
+def test_readPGF_NGF(NGF):
+    with pytest.raises(pgf.PGFError):
+        pgf.readPGF("./basic.ngf")
+
+# bootNGF
+
+@pytest.fixture(scope="module")
+def NGF():
+    return pgf.bootNGF("../haskell/tests/basic.pgf", "./basic.ngf")
+
+def test_bootNGF_non_existant():
+    with pytest.raises(FileNotFoundError):
+        pgf.bootNGF("../haskell/tests/abc.pgf", "./abc.ngf")
+
+def test_bootNGF_GF():
+    with pytest.raises(pgf.PGFError):
+        pgf.bootNGF("../haskell/tests/basic.gf", "./abc.ngf")
+
+@pytest.mark.skip(reason="Bug in runtime")
+def test_bootNGF_NGF(NGF):
+    with pytest.raises(pgf.PGFError):
+        pgf.bootNGF("./basic.ngf", "./abc.ngf")
+
+def test_bootNGF_existing(NGF):
+    with pytest.raises(FileExistsError):
+        pgf.bootNGF("../haskell/tests/basicp.gf", "./basic.ngf")
+
+# readNGF
+
+@pytest.mark.skip(reason="Bug in runtime")
+def test_readNGF_non_existant():
+    with pytest.raises(FileNotFoundError):
+        pgf.readNGF("./abc.ngf")
+
+@pytest.mark.skip(reason="Bug in runtime")
+def test_readNGF_GF():
+    with pytest.raises(pgf.PGFError):
+        pgf.readNGF("../haskell/tests/basic.gf")
+
+@pytest.mark.skip(reason="Bug in runtime")
+def test_readNGF_PGF():
+    with pytest.raises(pgf.PGFError):
+        pgf.readNGF("../haskell/tests/basic.pgf")
+
+def test_readNGF(NGF):
+    pgf.readNGF("./basic.ngf")
+    # TODO assert load worked