Skip to content
Snippets Groups Projects
Commit 66f014d0 authored by chrg's avatar chrg
Browse files

Since last time

parent 0375c910
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/env python3
""" The jpamb evaluator
"""
import click
import subprocess
import sys
from dataclasses import dataclass
prim = bool | int
@dataclass
class Case:
methodid: str
input: str
result: str
def rebuild():
subprocess.call(["mvn", "compile"])
def runtime(args, enable_assertions=False, **kwargs):
pargs = ["java", "-cp", "target/classes/"]
if enable_assertions:
pargs += ["-ea"]
pargs += ["jpamb.Runtime"]
pargs += args
return subprocess.check_output(pargs, text=True, **kwargs)
def getcases():
import csv
for r in sorted(
csv.reader(runtime([]).splitlines(), delimiter=" ", skipinitialspace=True)
):
yield Case(r[0], *r[1].split(" -> "))
@click.group
def cli():
"""The jpamb evaluator"""
@cli.command
def cases():
"""Get a list of cases to test"""
for c in getcases():
print(c)
@cli.command
@click.option("--timeout", default=0.5)
@click.argument(
"CMD",
nargs=-1,
)
def test(cmd, timeout):
"""Check that all cases are valid"""
if not cmd:
cmd = ["java", "-cp", "target/classes", "-ea", "jpamb.Runtime"]
rebuild()
cases = list(getcases())
failed = []
for c in cases:
print(f"=" * 80)
print(f"{c.methodid} with {c.input}")
print()
sys.stdout.flush()
try:
cp = subprocess.run(
cmd + [c.methodid, c.input],
text=True,
stderr=sys.stdout,
stdout=subprocess.PIPE,
timeout=timeout,
check=True,
)
result = cp.stdout.strip()
success = result == c.result
print()
print(f"Got {result} which is {success}")
except subprocess.CalledProcessError:
print()
print(f"Process failed.")
success = False
except subprocess.TimeoutExpired:
success = "*" == c.result
print()
print(f"Timed out after {timeout}s which is {success}")
if not success:
failed += [c]
print(f"=" * 80)
print()
if failed:
print(f"Failed on {len(failed)}/{len(cases)}")
else:
print(f"Sucessfully handled {len(cases)} cases")
@cli.command
def evaluate():
"""Check that all cases are valid"""
for c in getcases():
try:
result = runtime(
[c.methodid, c.input],
enable_assertions=True,
timeout=0.5,
).strip()
except subprocess.TimeoutExpired:
result = "*"
print(c, result == c.result)
if __name__ == "__main__":
cli()
......@@ -21,7 +21,7 @@
jdt-language-server
jdk
maven
(python3.withPackages (p: with p; []))
(python3.withPackages (p: with p; [click pandas]))
];
};
};
......
package jpamb;
import java.lang.reflect.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.Arrays;
import java.util.List;
import java.util.regex.*;
import java.util.stream.Stream;
import jpamb.utils.*;
import jpamb.utils.CaseContent.ResultType;
import jpamb.cases.*;
/**
* The runtime method runs a single test-case and print the result or the
* exeception.
*/
public class Runtime {
static List<Class<?>> caseclasses = List.of(
Simple.class,
Loops.class);
public static Case[] cases(Method m) {
var cases = m.getAnnotation(Cases.class);
......@@ -22,51 +31,91 @@ public class Runtime {
}
}
public static void main(String[] args) throws ClassNotFoundException, InterruptedException {
var mths = Stream.of(Simple.class, Loops.class).flatMap(c -> Stream.of(c.getMethods())).toList();
public static void printType(Class<?> c, StringBuilder b) {
if (c.equals(void.class)) {
b.append("V");
} else if (c.equals(int.class)) {
b.append("I");
} else if (c.equals(boolean.class)) {
b.append("Z");
} else if (c.equals(double.class)) {
b.append("D");
} else if (c.equals(float.class)) {
b.append("F");
} else if (c.equals(char.class)) {
b.append("C");
} else {
throw new RuntimeException("Unknown type:" + c.toString());
}
}
public static String printMethodSignature(Method m) {
StringBuilder b = new StringBuilder();
b.append("(");
for (Class<?> c : m.getParameterTypes()) {
printType(c, b);
}
b.append(")");
printType(m.getReturnType(), b);
return b.toString();
}
public static Class<?>[] parseMethodSignature(String s) {
Class<?>[] params = new Class[s.length()];
for (int i = 0; i < s.length(); i++) {
switch (s.charAt(i)) {
case 'I' -> {
params[i] = int.class;
break;
}
case 'Z' -> {
params[i] = boolean.class;
break;
}
}
}
return params;
}
public static void main(String[] args)
throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException {
if (args.length == 0) {
var mths = caseclasses.stream().flatMap(c -> Stream.of(c.getMethods())).toList();
for (Method m : mths) {
for (Case c : cases(m)) {
CaseContent content = CaseContent.parse(c.value());
String sig = printMethodSignature(m);
String id = m.getDeclaringClass().getName() + "." + m.getName() + ":" + sig;
if (!Modifier.isStatic(m.getModifiers())) {
System.out.println("Method is not static");
continue;
throw new RuntimeException("Expected " + id + " to be static");
}
String id = m.getDeclaringClass().getName() + "." + m.getName() + content + ":";
System.out.printf("%-80s", id);
System.out.flush();
final AtomicReference<Throwable> atom = new AtomicReference<>();
Thread t = new Thread(() -> {
try {
m.invoke(null, content.params());
} catch (InvocationTargetException e) {
atom.set(e.getCause());
} catch (IllegalAccessException e) {
atom.set(e);
System.out.printf("%-60s \"%s\"%n", id, content);
}
});
t.start();
t.join(100);
if (t.isAlive())
t.stop();
Throwable error = atom.get();
if (t.isAlive() && error == null) {
error = new TimeoutException();
}
String message;
if (error == null) {
message = "did not produce error";
} else if (content.result().expectThrows(error.getClass())) {
message = "success";
} else {
message = error.toString();
return;
}
String thecase = args[0];
Pattern pattern = Pattern.compile("(.*)\\.([^.(]*):\\((.*)\\)(.*)");
Matcher matcher = pattern.matcher(thecase);
if (matcher.find()) {
String cls = matcher.group(1);
String mth = matcher.group(2);
String prams = matcher.group(3);
Method m = Class.forName(cls).getMethod(mth, parseMethodSignature(prams));
if (!Modifier.isStatic(m.getModifiers())) {
throw new RuntimeException("Expected " + pattern + " to be static");
}
for (int i = 1; i < args.length; i++) {
Object[] params = CaseContent.parseParams(args[i]);
System.err.printf("Running %s with %s%n", m, Arrays.toString(params));
try {
m.invoke(null, params);
} catch (InvocationTargetException e) {
System.out.println(ResultType.fromThrowable(e.getCause()));
return;
}
System.out.printf("%s%n", message);
}
System.out.println(ResultType.SUCCESS);
}
}
}
......@@ -35,7 +35,7 @@ public class Simple {
}
@Case("(0, 0) -> divide by zero")
public static double divideZeroByZero(int a, int b) {
public static int divideZeroByZero(int a, int b) {
return a / b;
}
......
......@@ -19,16 +19,13 @@ public record CaseContent(
return "(" + String.join(", ", sparams) + ") -> " + result.toString();
}
public static CaseContent parse(String string) {
Pattern pattern = Pattern.compile("\\(([^)]*)\\)\\s*->\\s*(.+)");
Matcher matcher = pattern.matcher(string);
static Pattern paramPattern = Pattern.compile("\\(([^)]*)\\)");
// Parse the expression
public static Object[] parseParams(String string) {
Matcher matcher = paramPattern.matcher(string);
if (matcher.find()) {
String args = matcher.group(1);
String result = matcher.group(2);
ArrayList<Object> list = new ArrayList<>();
try (Scanner sc = new Scanner(args)) {
try (Scanner sc = new Scanner(matcher.group(1))) {
sc.useLocale(Locale.US);
sc.useDelimiter(" *, *");
while (sc.hasNext()) {
......@@ -39,12 +36,25 @@ public record CaseContent(
} else {
String var = sc.next();
if (!var.equals(",")) {
throw new RuntimeException("Invalid case: " + string + " // unexpected " + var);
throw new RuntimeException("Invalid parameter: " + string + " // unexpected " + var);
}
}
}
}
return new CaseContent(list.toArray(), ResultType.parse(result));
return list.toArray();
} else {
throw new RuntimeException(string + " is not a paramater list");
}
}
public static CaseContent parse(String string) {
Pattern pattern = Pattern.compile("(\\([^)]*\\))\\s*->\\s*(.+)");
Matcher matcher = pattern.matcher(string);
// Parse the expression
if (matcher.find()) {
String args = matcher.group(1);
String result = matcher.group(2);
return new CaseContent(parseParams(args), ResultType.parse(result));
} else {
throw new RuntimeException("Invalid case: " + string);
}
......@@ -53,6 +63,7 @@ public record CaseContent(
public static enum ResultType {
DIVIDE_BY_ZERO,
ASSERTION_ERROR,
SUCCESS,
NON_TERMINATION;
public static ResultType parse(String string) {
......@@ -75,6 +86,8 @@ public record CaseContent(
return "assertion error";
case NON_TERMINATION:
return "*";
case SUCCESS:
return "ok";
default:
throw new RuntimeException("Unexpected");
}
......@@ -92,5 +105,17 @@ public record CaseContent(
throw new RuntimeException("Unexpected");
}
}
public static ResultType fromThrowable(Throwable cause) {
if (cause instanceof ArithmeticException) {
return DIVIDE_BY_ZERO;
} else if (cause instanceof AssertionError) {
return ASSERTION_ERROR;
} else if (cause instanceof TimeoutException) {
return NON_TERMINATION;
} else {
throw new RuntimeException("Unexpected");
}
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment