diff --git a/evaluate.py b/evaluate.py
new file mode 100755
index 0000000000000000000000000000000000000000..7047daca48f528370513c27121e51f3de3046f54
--- /dev/null
+++ b/evaluate.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+""" The jpamb evaluator
+"""
+
+import click
+import subprocess
+import sys
+from dataclasses import dataclass
+
+prim = bool | int
+
+
+@dataclass
+class Case:
+ methodid: str
+ input: str
+ result: str
+
+
+def rebuild():
+ subprocess.call(["mvn", "compile"])
+
+
+def runtime(args, enable_assertions=False, **kwargs):
+ pargs = ["java", "-cp", "target/classes/"]
+
+ if enable_assertions:
+ pargs += ["-ea"]
+
+ pargs += ["jpamb.Runtime"]
+ pargs += args
+
+ return subprocess.check_output(pargs, text=True, **kwargs)
+
+
+def getcases():
+ import csv
+
+ for r in sorted(
+ csv.reader(runtime([]).splitlines(), delimiter=" ", skipinitialspace=True)
+ ):
+ yield Case(r[0], *r[1].split(" -> "))
+
+
+@click.group
+def cli():
+ """The jpamb evaluator"""
+
+
+@cli.command
+def cases():
+ """Get a list of cases to test"""
+ for c in getcases():
+ print(c)
+
+
+@cli.command
+@click.option("--timeout", default=0.5)
+@click.argument(
+ "CMD",
+ nargs=-1,
+)
+def test(cmd, timeout):
+ """Check that all cases are valid"""
+ if not cmd:
+ cmd = ["java", "-cp", "target/classes", "-ea", "jpamb.Runtime"]
+
+ rebuild()
+
+ cases = list(getcases())
+ failed = []
+ for c in cases:
+ print(f"=" * 80)
+ print(f"{c.methodid} with {c.input}")
+ print()
+ sys.stdout.flush()
+
+ try:
+ cp = subprocess.run(
+ cmd + [c.methodid, c.input],
+ text=True,
+ stderr=sys.stdout,
+ stdout=subprocess.PIPE,
+ timeout=timeout,
+ check=True,
+ )
+ result = cp.stdout.strip()
+ success = result == c.result
+ print()
+ print(f"Got {result} which is {success}")
+ except subprocess.CalledProcessError:
+ print()
+ print(f"Process failed.")
+ success = False
+ except subprocess.TimeoutExpired:
+ success = "*" == c.result
+ print()
+ print(f"Timed out after {timeout}s which is {success}")
+ if not success:
+ failed += [c]
+ print(f"=" * 80)
+ print()
+
+ if failed:
+ print(f"Failed on {len(failed)}/{len(cases)}")
+ else:
+ print(f"Sucessfully handled {len(cases)} cases")
+
+
+@cli.command
+def evaluate():
+ """Check that all cases are valid"""
+ for c in getcases():
+ try:
+ result = runtime(
+ [c.methodid, c.input],
+ enable_assertions=True,
+ timeout=0.5,
+ ).strip()
+ except subprocess.TimeoutExpired:
+ result = "*"
+ print(c, result == c.result)
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/flake.nix b/flake.nix
index a7f5adf02f092648ad75cd546e260ac5a4312b34..42f26198021bb3cec468942d6029c37c599b4410 100644
--- a/flake.nix
+++ b/flake.nix
@@ -21,7 +21,7 @@
jdt-language-server
jdk
maven
- (python3.withPackages (p: with p; []))
+ (python3.withPackages (p: with p; [click pandas]))
];
};
};
diff --git a/src/main/java/jpamb/Runtime.java b/src/main/java/jpamb/Runtime.java
index d895fa904ba11b7b1212f4f55157861e68399322..3089b525d778b3b86254b9a9282dcc8e092080bc 100644
--- a/src/main/java/jpamb/Runtime.java
+++ b/src/main/java/jpamb/Runtime.java
@@ -1,14 +1,23 @@
package jpamb;
import java.lang.reflect.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.*;
import java.util.stream.Stream;
import jpamb.utils.*;
+import jpamb.utils.CaseContent.ResultType;
import jpamb.cases.*;
+/**
+ * The runtime method runs a single test-case and print the result or the
+ * exeception.
+ */
public class Runtime {
+ static List<Class<?>> caseclasses = List.of(
+ Simple.class,
+ Loops.class);
public static Case[] cases(Method m) {
var cases = m.getAnnotation(Cases.class);
@@ -22,51 +31,91 @@ public class Runtime {
}
}
- public static void main(String[] args) throws ClassNotFoundException, InterruptedException {
- var mths = Stream.of(Simple.class, Loops.class).flatMap(c -> Stream.of(c.getMethods())).toList();
- for (Method m : mths) {
- for (Case c : cases(m)) {
- CaseContent content = CaseContent.parse(c.value());
+ public static void printType(Class<?> c, StringBuilder b) {
+ if (c.equals(void.class)) {
+ b.append("V");
+ } else if (c.equals(int.class)) {
+ b.append("I");
+ } else if (c.equals(boolean.class)) {
+ b.append("Z");
+ } else if (c.equals(double.class)) {
+ b.append("D");
+ } else if (c.equals(float.class)) {
+ b.append("F");
+ } else if (c.equals(char.class)) {
+ b.append("C");
+ } else {
+ throw new RuntimeException("Unknown type:" + c.toString());
+ }
+ }
+
+ public static String printMethodSignature(Method m) {
+ StringBuilder b = new StringBuilder();
+ b.append("(");
+ for (Class<?> c : m.getParameterTypes()) {
+ printType(c, b);
+ }
+ b.append(")");
+ printType(m.getReturnType(), b);
+ return b.toString();
+ }
- if (!Modifier.isStatic(m.getModifiers())) {
- System.out.println("Method is not static");
- continue;
+ public static Class<?>[] parseMethodSignature(String s) {
+ Class<?>[] params = new Class[s.length()];
+ for (int i = 0; i < s.length(); i++) {
+ switch (s.charAt(i)) {
+ case 'I' -> {
+ params[i] = int.class;
+ break;
}
+ case 'Z' -> {
+ params[i] = boolean.class;
+ break;
+ }
+ }
+ }
+ return params;
+ }
- String id = m.getDeclaringClass().getName() + "." + m.getName() + content + ":";
- System.out.printf("%-80s", id);
- System.out.flush();
- final AtomicReference<Throwable> atom = new AtomicReference<>();
- Thread t = new Thread(() -> {
- try {
- m.invoke(null, content.params());
- } catch (InvocationTargetException e) {
- atom.set(e.getCause());
- } catch (IllegalAccessException e) {
- atom.set(e);
+ public static void main(String[] args)
+ throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException {
+ if (args.length == 0) {
+ var mths = caseclasses.stream().flatMap(c -> Stream.of(c.getMethods())).toList();
+ for (Method m : mths) {
+ for (Case c : cases(m)) {
+ CaseContent content = CaseContent.parse(c.value());
+ String sig = printMethodSignature(m);
+ String id = m.getDeclaringClass().getName() + "." + m.getName() + ":" + sig;
+ if (!Modifier.isStatic(m.getModifiers())) {
+ throw new RuntimeException("Expected " + id + " to be static");
}
- });
- t.start();
- t.join(100);
-
- if (t.isAlive())
- t.stop();
-
- Throwable error = atom.get();
- if (t.isAlive() && error == null) {
- error = new TimeoutException();
+ System.out.printf("%-60s \"%s\"%n", id, content);
}
-
- String message;
- if (error == null) {
- message = "did not produce error";
- } else if (content.result().expectThrows(error.getClass())) {
- message = "success";
- } else {
- message = error.toString();
+ }
+ return;
+ }
+ String thecase = args[0];
+ Pattern pattern = Pattern.compile("(.*)\\.([^.(]*):\\((.*)\\)(.*)");
+ Matcher matcher = pattern.matcher(thecase);
+ if (matcher.find()) {
+ String cls = matcher.group(1);
+ String mth = matcher.group(2);
+ String prams = matcher.group(3);
+ Method m = Class.forName(cls).getMethod(mth, parseMethodSignature(prams));
+ if (!Modifier.isStatic(m.getModifiers())) {
+ throw new RuntimeException("Expected " + pattern + " to be static");
+ }
+ for (int i = 1; i < args.length; i++) {
+ Object[] params = CaseContent.parseParams(args[i]);
+ System.err.printf("Running %s with %s%n", m, Arrays.toString(params));
+ try {
+ m.invoke(null, params);
+ } catch (InvocationTargetException e) {
+ System.out.println(ResultType.fromThrowable(e.getCause()));
+ return;
}
- System.out.printf("%s%n", message);
}
+ System.out.println(ResultType.SUCCESS);
}
}
}
diff --git a/src/main/java/jpamb/cases/Simple.java b/src/main/java/jpamb/cases/Simple.java
index fe2fb8cca61cdef3a3858f9865f7ec71b759c977..0caf225310bd9bb679bc8a3a8508df27da445af8 100644
--- a/src/main/java/jpamb/cases/Simple.java
+++ b/src/main/java/jpamb/cases/Simple.java
@@ -35,7 +35,7 @@ public class Simple {
}
@Case("(0, 0) -> divide by zero")
- public static double divideZeroByZero(int a, int b) {
+ public static int divideZeroByZero(int a, int b) {
return a / b;
}
diff --git a/src/main/java/jpamb/utils/CaseContent.java b/src/main/java/jpamb/utils/CaseContent.java
index 39b5299842354a42f503c5427c1e379c55b7bf83..9e0ffe1198762b595e2220837f9dd8644aa96dcb 100644
--- a/src/main/java/jpamb/utils/CaseContent.java
+++ b/src/main/java/jpamb/utils/CaseContent.java
@@ -19,16 +19,13 @@ public record CaseContent(
return "(" + String.join(", ", sparams) + ") -> " + result.toString();
}
- public static CaseContent parse(String string) {
- Pattern pattern = Pattern.compile("\\(([^)]*)\\)\\s*->\\s*(.+)");
- Matcher matcher = pattern.matcher(string);
+ static Pattern paramPattern = Pattern.compile("\\(([^)]*)\\)");
- // Parse the expression
+ public static Object[] parseParams(String string) {
+ Matcher matcher = paramPattern.matcher(string);
if (matcher.find()) {
- String args = matcher.group(1);
- String result = matcher.group(2);
ArrayList<Object> list = new ArrayList<>();
- try (Scanner sc = new Scanner(args)) {
+ try (Scanner sc = new Scanner(matcher.group(1))) {
sc.useLocale(Locale.US);
sc.useDelimiter(" *, *");
while (sc.hasNext()) {
@@ -39,12 +36,25 @@ public record CaseContent(
} else {
String var = sc.next();
if (!var.equals(",")) {
- throw new RuntimeException("Invalid case: " + string + " // unexpected " + var);
+ throw new RuntimeException("Invalid parameter: " + string + " // unexpected " + var);
}
}
}
}
- return new CaseContent(list.toArray(), ResultType.parse(result));
+ return list.toArray();
+ } else {
+ throw new RuntimeException(string + " is not a paramater list");
+ }
+ }
+
+ public static CaseContent parse(String string) {
+ Pattern pattern = Pattern.compile("(\\([^)]*\\))\\s*->\\s*(.+)");
+ Matcher matcher = pattern.matcher(string);
+ // Parse the expression
+ if (matcher.find()) {
+ String args = matcher.group(1);
+ String result = matcher.group(2);
+ return new CaseContent(parseParams(args), ResultType.parse(result));
} else {
throw new RuntimeException("Invalid case: " + string);
}
@@ -53,6 +63,7 @@ public record CaseContent(
public static enum ResultType {
DIVIDE_BY_ZERO,
ASSERTION_ERROR,
+ SUCCESS,
NON_TERMINATION;
public static ResultType parse(String string) {
@@ -75,6 +86,8 @@ public record CaseContent(
return "assertion error";
case NON_TERMINATION:
return "*";
+ case SUCCESS:
+ return "ok";
default:
throw new RuntimeException("Unexpected");
}
@@ -92,5 +105,17 @@ public record CaseContent(
throw new RuntimeException("Unexpected");
}
}
+
+ public static ResultType fromThrowable(Throwable cause) {
+ if (cause instanceof ArithmeticException) {
+ return DIVIDE_BY_ZERO;
+ } else if (cause instanceof AssertionError) {
+ return ASSERTION_ERROR;
+ } else if (cause instanceof TimeoutException) {
+ return NON_TERMINATION;
+ } else {
+ throw new RuntimeException("Unexpected");
+ }
+ }
}
}