Serve plain Python: sessions declare the parameters

Jython's analysis and the Python server beside it now read the same
text, which the session makes in JythonDialect: parameters are
declared as instances of their classes (e.g. img = net.imglib2.img.Img()),
which both understand. So the server no longer reads #@ lines, and
PythonCompanion no longer transforms scripts for jedi.

Jython's analysis now resolves classes by their full names, and the
stubs follow imports separated by semicolons.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: gselzer

src/main/java/org/scijava/plugins/scripting/jython/JavaStubs.java

@@ -182,7 +182,8 @@ public CompletableFuture<Void> ensure(final Collection<String> packages) {
 	 */
 	public static Set<String> imports(final String script) {
 		final Set<String> packages = new TreeSet<>();
-		for (final String line : script.split("\n")) {
+		// NB: Statements on one line are separated by semicolons.
+		for (final String line : script.split("[\n;]")) {
 			final String l = line.trim();
 			String module = null;
 			if (l.startsWith("from ")) {

src/main/java/org/scijava/plugins/scripting/jython/JythonDialect.java

@@ -38,14 +38,12 @@
 
 /**
  * How Jython scripts see their script parameters: as the Java objects
- * themselves (primitives boxed), e.g. {@code image: net.imglib2.img.Img}; or
- * as Python values, where Jython converts them (e.g. strings).
+ * themselves (primitives boxed), e.g. {@code net.imglib2.img.Img}.
  * <p>
- * Run-time types are Java class names (e.g. {@code java.lang.String}), as
- * Jython's own completion needs them; declarations are Python, for a Python
- * language server knowing the Java packages from stubs (see
- * {@link JavaStubs}), e.g. {@code import net.imglib2.img; image:
- * net.imglib2.img.Img; name: str}.
+ * Each is declared as an instance of its class, in code both Jython's own
+ * analysis (Python 2) and a Python language server knowing the Java packages
+ * from stubs (see {@link JavaStubs}) read: e.g. {@code import net.imglib2.img;
+ * image = net.imglib2.img.Img()}. (Arrays are lists.)
  * </p>
  *
  * @author Gabriel Selzer
@@ -59,7 +57,8 @@ public String runtimeType(final Class<?> type) {
 
 	@Override
 	public String declare(final String name, final String type) {
-		return name + ": " + pythonType(type);
+		return name + " = " + (type.startsWith("[") ? "list" : type.replace('$',
+			'.')) + "()";
 	}
 
 	@Override
@@ -73,38 +72,12 @@ public List<String> imports(final Collection<String> types) {
 	static Set<String> packages(final Collection<String> types) {
 		final Set<String> packages = new TreeSet<>();
 		for (final String t : types) {
-			if (pythonType(t).equals(t.replace('$', '.')) && t.lastIndexOf(
-				'.') > 0) packages.add(t.substring(0, t.lastIndexOf('.')));
+			if (!t.startsWith("[") && t.lastIndexOf('.') > 0) packages.add(t
+				.substring(0, t.lastIndexOf('.')));
 		}
 		return packages;
 	}
 
-	/**
-	 * The Python type of a Java class name, as Jython converts its values:
-	 * e.g. {@code str} for {@code java.lang.String}.
-	 */
-	static String pythonType(final String type) {
-		switch (type) {
-			case "java.lang.String":
-			case "java.lang.Character":
-				return "str";
-			case "java.lang.Integer":
-			case "java.lang.Long":
-			case "java.lang.Short":
-			case "java.lang.Byte":
-			case "java.math.BigInteger":
-				return "int";
-			case "java.lang.Double":
-			case "java.lang.Float":
-			case "java.math.BigDecimal":
-				return "float";
-			case "java.lang.Boolean":
-				return "bool";
-			default:
-				return type.startsWith("[") ? "list" : type.replace('$', '.');
-		}
-	}
-
 	/** Maps primitive types to their wrappers, whose members can be listed. */
 	private static Class<?> boxed(final Class<?> type) {
 		if (!type.isPrimitive()) return type;

src/main/java/org/scijava/plugins/scripting/jython/JythonLanguageServer.java

@@ -58,7 +58,6 @@
 import org.python.core.PyObject;
 import org.python.core.PyString;
 import org.scijava.code.lsp.jvm.ClassIndex;
-import org.scijava.code.lsp.LanguageServerService;
 import org.scijava.code.lsp.jvm.Callable;
 import org.scijava.code.lsp.Environment;
 import org.scijava.code.lsp.jvm.ScriptLanguageServer;
@@ -83,40 +82,37 @@
  */
 public class JythonLanguageServer extends ScriptLanguageServer {
 
-	/** How scripts see their parameters: as the Java objects themselves. */
+	/**
+	 * How scripts see their parameters: as the Java objects themselves,
+	 * declared in the text (see {@link JythonServerPlugin#dialect()}).
+	 */
 	static final JythonDialect DIALECT = new JythonDialect();
 
 	static {
 		// Warm the class index in the background, as completion needs it.
 		new Thread(ClassIndex::ensureCache, "Jython-ClassIndex-warmup").start();
 	}
 
-	private final LanguageServerService servers;
 	private final JythonAutoCompletions engine = new JythonAutoCompletions();
 
 	/** A Python language server beside Jython's analysis, or null if none. */
 	private final PythonCompanion python;
 
 	/**
-	 * @param servers Reads the scripts' parameters.
 	 * @param environment Where the scripts run (e.g. a live interpreter).
 	 */
-	public JythonLanguageServer(final LanguageServerService servers,
-		final Environment environment)
-	{
-		this(servers, environment, null);
+	public JythonLanguageServer(final Environment environment) {
+		this(environment, null);
 	}
 
 	/**
-	 * @param servers Reads the scripts' parameters.
 	 * @param environment Where the scripts run (e.g. a live interpreter).
 	 * @param python A Python language server beside Jython's analysis, or null.
 	 */
-	JythonLanguageServer(final LanguageServerService servers,
-		final Environment environment, final PythonCompanion python)
+	JythonLanguageServer(final Environment environment,
+		final PythonCompanion python)
 	{
 		super(environment);
-		this.servers = servers;
 		this.python = python;
 	}
 
@@ -310,24 +306,16 @@ private JythonAutoCompletions.Result completionsAt(final String text,
 			'\n') + 1);
 		return engine.completionsFor(textToCaret.substring(0, textToCaret
 			.length() - lastLine.length()), lastLine, alreadyEnteredText(lastLine),
-			predefinedVariables(text));
+			liveVariables());
 	}
 
 	/**
-	 * Gets the variables defined before the script's code runs: script
-	 * parameters declared via {@code #@} lines, and, when completing in a live
-	 * interpreter, its bindings (which reflect actual runtime values, so they
-	 * take precedence).
+	 * Gets the variables defined before the code runs, when completing in a
+	 * live interpreter: its bindings. (A script's parameters are declared in
+	 * its text.)
 	 */
-	private Map<String, DotAutocompletions> predefinedVariables(
-		final String text)
-	{
+	private Map<String, DotAutocompletions> liveVariables() {
 		final Map<String, DotAutocompletions> vars = new LinkedHashMap<>();
-		if (servers != null) {
-			servers.scriptParameters(text).forEach((name,
-				type) -> vars.put(name, new VarDotAutocompletions(DIALECT.runtimeType(
-					type))));
-		}
 		final ScriptContext live = environment().live();
 		final Bindings bindings = live == null ? null : live.getBindings(
 			ScriptContext.ENGINE_SCOPE);

src/main/java/org/scijava/plugins/scripting/jython/JythonScriptParser.java

@@ -404,6 +404,13 @@ static public DotAutocompletions parseRight(final PyObject right, final Scope sc
 			// gray8 = IJ.getImage().GRAY8
 			final Attribute attr = (Attribute)right;
 			final DotAutocompletions da = parseRight(attr.getValue(), scope);
+			// e.g. a class by its full name, after a package (or nothing known)
+			// img = net.imglib2.img.Img()
+			if (DotAutocompletions.EMPTY == da || da instanceof StaticDotAutocompletions) {
+				final String fullName = javaClass(attr);
+				if (null != fullName)
+					return new StaticDotAutocompletions(fullName);
+			}
 			if (DotAutocompletions.EMPTY == da)
 				return da;
 			final String name = attr.getInternalAttr();
@@ -465,4 +472,32 @@ static public DotAutocompletions parseRight(final PyObject right, final Scope sc
 
 		return DotAutocompletions.EMPTY;
 	}
+
+	/**
+	 * The Java class a dotted name (e.g. {@code java.util.Map.Entry}) is, by
+	 * its full name; or null if none (e.g. it starts with a variable).
+	 */
+	static private String javaClass(final Attribute attr) {
+		final StringBuilder name = new StringBuilder(attr.getInternalAttr());
+		PyObject node = attr.getValue();
+		while (node instanceof Attribute) {
+			name.insert(0, ((Attribute)node).getInternalAttr() + ".");
+			node = ((Attribute)node).getValue();
+		}
+		if (!(node instanceof Name)) return null;
+		name.insert(0, ((Name)node).getInternalId() + ".");
+		// NB: A package starts lowercase; a class, uppercase.
+		if (!Character.isLowerCase(name.charAt(0))) return null;
+		String candidate = name.toString();
+		while (true) {
+			try {
+				return Class.forName(candidate).getName();
+			} catch (ClassNotFoundException | LinkageError e) {
+				// Perhaps a nested class: e.g. java.util.Map$Entry.
+				final int dot = candidate.lastIndexOf('.');
+				if (dot < 0 || !Character.isUpperCase(candidate.charAt(candidate.lastIndexOf('.', dot - 1) + 1))) return null;
+				candidate = candidate.substring(0, dot) + "$" + candidate.substring(dot + 1);
+			}
+		}
+	}
 }

src/main/java/org/scijava/plugins/scripting/jython/JythonServerPlugin.java

@@ -35,14 +35,16 @@
 import org.scijava.code.lsp.LanguageServerService;
 import org.scijava.code.lsp.Environment;
 import org.scijava.code.lsp.LanguageServerPlugin;
+import org.scijava.code.lsp.script.ScriptDialect;
 import org.scijava.plugin.Parameter;
 import org.scijava.plugin.Plugin;
 import org.scijava.script.ScriptLanguage;
 
 /**
  * Offers Jython's in-process language server (see
  * {@link JythonLanguageServer}), with a Python language server beside it if
- * one is offered (see {@link PythonCompanion}).
+ * one is offered (see {@link PythonCompanion}); both see scripts as plain
+ * Python (see {@link JythonDialect}).
  *
  * @author Gabriel Selzer
  */
@@ -62,8 +64,13 @@ public boolean supports(final ScriptLanguage language) {
 
 	@Override
 	public LanguageServer create(final Environment environment) {
-		return new JythonLanguageServer(servers, environment, PythonCompanion
-			.launch(servers, stubs()));
+		return new JythonLanguageServer(environment, PythonCompanion.launch(
+			servers, stubs()));
+	}
+
+	@Override
+	public ScriptDialect dialect() {
+		return JythonLanguageServer.DIALECT;
 	}
 
 	/** Sets the stubs, e.g. to write them elsewhere in tests. */

src/main/java/org/scijava/plugins/scripting/jython/PythonCompanion.java

@@ -33,12 +33,9 @@
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import org.eclipse.lsp4j.CompletionItem;
 import org.eclipse.lsp4j.CompletionList;
@@ -57,15 +54,14 @@
 import org.eclipse.lsp4j.services.TextDocumentService;
 import org.scijava.code.lsp.Environment;
 import org.scijava.code.lsp.LanguageServerService;
-import org.scijava.code.lsp.script.ScriptDocument;
-import org.scijava.code.lsp.script.TransformingLanguageServer;
 
 /**
  * A Python language server (e.g. jedi-language-server, if the
  * {@code appose-python} scripting plugin offers one) beside Jython's own
  * analysis (see {@link JythonLanguageServer}): for Python modules, and for
- * Java classes, from Python stubs of the Java packages a script imports and
- * its parameters' types (see {@link JavaStubs}, written in the background).
+ * Java classes, from Python stubs of the Java packages a script imports
+ * (including those of its parameters' types, declared in its text; see
+ * {@link JavaStubs}, written in the background).
  * <p>
  * The server analyzes scripts as CPython (Jython's standard library is Python
  * 2's, but mostly the same): so its completions follow Jython's own, it is
@@ -77,36 +73,30 @@
  */
 final class PythonCompanion {
 
-	/** The server, seeing scripts as plain Python. */
+	/** The server. */
 	private final LanguageServer server;
 	private final JavaStubs stubs;
-	private final Function<String, Map<String, Class<?>>> parameters;
 
-	private PythonCompanion(final LanguageServer python, final JavaStubs stubs,
-		final Function<String, Map<String, Class<?>>> parameters)
-	{
+	private PythonCompanion(final LanguageServer python, final JavaStubs stubs) {
 		this.stubs = stubs;
-		this.parameters = parameters;
 		// NB: No client: the server's problems are not asked for.
-		server = new TransformingLanguageServer(python, text -> ScriptDocument.of(
-			text, parameters.apply(text), JythonLanguageServer.DIALECT, null));
+		server = python;
 		server.initialize(new InitializeParams()).thenRun(() -> server
 			.initialized(new InitializedParams()));
 	}
 
 	/**
 	 * Gets a Python server beside Jython's, or null if none is offered.
 	 *
-	 * @param servers Offers the server, and reads the scripts' parameters.
+	 * @param servers Offers the server.
 	 * @param stubs The stubs of Java packages, for the server to search.
 	 */
 	static PythonCompanion launch(final LanguageServerService servers,
 		final JavaStubs stubs)
 	{
 		if (servers == null) return null;
 		final LanguageServer python = servers.launch("python", environment(stubs));
-		return python == null ? null : new PythonCompanion(python, stubs,
-			servers::scriptParameters);
+		return python == null ? null : new PythonCompanion(python, stubs);
 	}
 
 	/** Where the server runs: its own Python, searching the Java stubs. */
@@ -192,14 +182,9 @@ private TextDocumentService documents() {
 		return server.getTextDocumentService();
 	}
 
-	/** Writes the stubs the script needs, in the background. */
+	/** Writes the stubs of the packages the script imports, in the background. */
 	private void ensureStubs(final String text) {
-		final Set<String> packages = new TreeSet<>(JavaStubs.imports(text));
-		final ScriptDocument doc = ScriptDocument.of(text, parameters.apply(text),
-			JythonLanguageServer.DIALECT, null);
-		packages.addAll(JythonDialect.packages(doc.parameters().values().stream()
-			.filter(t -> t != null).collect(Collectors.toList())));
-		stubs.ensure(packages);
+		stubs.ensure(JavaStubs.imports(text));
 	}
 
 	/**

src/test/java/org/scijava/plugins/scripting/jython/JythonServerCompletionTest.java

@@ -118,15 +118,18 @@ public void testMerge() {
 	@Test
 	public void testDeclarations() {
 		final JythonDialect dialect = new JythonDialect();
-		assertEquals("img: net.imglib2.img.Img", dialect.declare("img",
+		assertEquals("img = net.imglib2.img.Img()", dialect.declare("img",
 			"net.imglib2.img.Img"));
-		assertEquals("n: int", dialect.declare("n", dialect.runtimeType(
-			int.class)));
-		assertEquals("e: java.util.Map.Entry", dialect.declare("e",
+		assertEquals("n = java.lang.Integer()", dialect.declare("n", dialect
+			.runtimeType(int.class)));
+		assertEquals("e = java.util.Map.Entry()", dialect.declare("e",
 			"java.util.Map$Entry"));
-		assertEquals(Arrays.asList("import java.util", "import net.imglib2.img"),
-			dialect.imports(Arrays.asList("net.imglib2.img.Img", "java.lang.String",
-				"java.util.Map$Entry")));
+		assertEquals("a = list()", dialect.declare("a", dialect.runtimeType(
+			double[].class)));
+		assertEquals(Arrays.asList("import java.lang", "import java.util",
+			"import net.imglib2.img"), dialect.imports(Arrays.asList(
+				"net.imglib2.img.Img", "java.lang.String", "java.util.Map$Entry",
+				"[D")));
 		assertTrue(JythonServerPlugin.isJython(jython));
 	}
 }